class DataRetrieverIntTestAlpha(IonIntegrationTestCase): def setUp(self): super(DataRetrieverIntTestAlpha,self).setUp() self._start_container() config = DotDict() config.bootstrap.processes.ingestion.module = 'ion.processes.data.ingestion.ingestion_worker_a' config.bootstrap.processes.replay.module = 'ion.processes.data.replay.replay_process_a' self.container.start_rel_from_url('res/deploy/r2dm.yml', config) self.datastore_name = 'test_datasets' self.datastore = self.container.datastore_manager.get_datastore(self.datastore_name, profile=DataStore.DS_PROFILE.SCIDATA) self.data_retriever = DataRetrieverServiceClient() self.dataset_management = DatasetManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() xs_dot_xp = CFG.core_xps.science_data try: self.XS, xp_base = xs_dot_xp.split('.') self.XP = '.'.join([get_sys_name(), xp_base]) except ValueError: raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_define_replay(self): # Create a dataset to work with dataset_id = self.dataset_management.create_dataset('fakestream', self.datastore_name) replay_id, stream_id = self.data_retriever.define_replay(dataset_id=dataset_id) # Verify that the replay instance was created replay = self.resource_registry.read(replay_id) pid = replay.process_id process = self.container.proc_manager.procs[pid] self.assertIsInstance(process,ReplayProcess, 'Incorrect process launched')
def build(resource_id, depth=0): ''' Constructs a JSONtree for the specified resource. The tree is built downward so all associations from this resource down are included. ''' rr_cli = ResourceRegistryServiceClient() resource = rr_cli.read(resource_id) root = jt(resource.name or resource._id) root.id = resource_id obj_list, assoc_list = rr_cli.find_objects(subject=resource_id,id_only=True) if not obj_list: root.leaf = True return root for obj,assoc in zip(obj_list,assoc_list): if obj and not (depth > tree_depth_max): root.add_child(build(obj,depth+1),assoc.p) return root
class EmailEventProcessor(object): """ A class that helps to get user subscribed to notifications """ def __init__(self): # the resource registry # TODO: This should be the client generated for the UNS (a process client) self.rr = ResourceRegistryServiceClient() def add_notification_for_user(self, new_notification=None, user_id=''): """ Add a notification to the user's list of subscribed notifications @param notification_request NotificationRequest @param user_id str """ user = self.rr.read(user_id) cond = True for item in user.variables: if item.has_key('name') and item['name'] == 'notifications': cond = False if cond: # this means that this dict is not already filled in the user info object dict = {'name' : 'notifications', 'value' : [new_notification]} user.variables.append(dict) else: # the user has previous notifications for item in user.variables: if item.has_key('name') and item['name'] == 'notifications': item['value'].append(new_notification) break #------------------------------------------------------------------------------------ # update the resource registry #------------------------------------------------------------------------------------ self.rr.update(user) return user
def build(resource_id, depth=0): ''' Constructs a JSONtree for the specified resource. The tree is built downward so all associations from this resource down are included. ''' rr_cli = ResourceRegistryServiceClient() resource = rr_cli.read(resource_id) root = jt(resource.name or resource._id) root.id = resource_id obj_list, assoc_list = rr_cli.find_objects(subject=resource_id, id_only=True) if not obj_list: root.leaf = True return root for obj, assoc in zip(obj_list, assoc_list): if obj and not (depth > tree_depth_max): root.add_child(build(obj, depth + 1), assoc.p) return root
class EmailEventProcessor(object): """ A class that helps to get user subscribed to notifications """ def __init__(self): # the resource registry self.rr = ResourceRegistryServiceClient() def add_notification_for_user(self, new_notification=None, user_id=''): """ Add a notification to the user's list of subscribed notifications @param notification_request NotificationRequest @param user_id str """ user = self.rr.read(user_id) cond = True for item in user.variables: if item.has_key('name') and item['name'] == 'notifications': cond = False if cond: # this means that this dict is not already filled in the user info object dict = {'name' : 'notifications', 'value' : [new_notification]} user.variables.append(dict) else: # the user has previous notifications for item in user.variables: if item.has_key('name') and item['name'] == 'notifications': item['value'].append(new_notification) break #------------------------------------------------------------------------------------ # update the resource registry #------------------------------------------------------------------------------------ self.rr.update(user) return user
class TestResourceRegistryAttachments(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) print 'started services' def test_resource_registry_blob_sanity(self): resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo")) MY_CONTENT = "the quick brown fox etc etc etc" #save att_id = self.RR.create_attachment(resource_id, IonObject(RT.Attachment, name="test.txt", content=MY_CONTENT, content_type="text/plain", keywords=["test1", "test2"], attachment_type=AttachmentType.BLOB)) #load attachment = self.RR.read_attachment(att_id, include_content=True) self.assertEqual("test.txt", attachment.name) self.assertEqual("text/plain", attachment.content_type) self.assertIn("test1", attachment.keywords) self.assertIn("test2", attachment.keywords) #content has changed; it's base64-encoded from what we put in self.assertEqual(MY_CONTENT, attachment.content) obj = self.RR.read(resource_id) self.assertEqual(obj.name, "foo") obj.name = "TheDudeAbides" obj = self.RR.update(obj) obj = self.RR.read(resource_id) self.assertEqual(obj.name, "TheDudeAbides") att = self.RR.find_attachments(resource_id) self.assertNotEqual(att, None) actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.RR.create(actor_identity_obj) user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.RR.create(user_info_obj) assoc_id, assoc_rev = self.RR.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertNotEqual(assoc_id, None) find_assoc = self.RR.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(find_assoc[0]._id == assoc_id) subj = self.RR.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) res_obj1 = self.RR.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) self.RR.delete_association(assoc_id) self.RR.delete_attachment(att_id) self.RR.delete(resource_id)
class IngestionManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url("res/deploy/r2deploy.yml") self.ingestion_management = IngestionManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.ingest_name = "basic" self.exchange = "testdata" @staticmethod def clean_subscriptions(): ingestion_management = IngestionManagementServiceClient() pubsub = PubsubManagementServiceClient() rr = ResourceRegistryServiceClient() ingestion_config_ids = ingestion_management.list_ingestion_configurations(id_only=True) for ic in ingestion_config_ids: assocs = rr.find_associations(subject=ic, predicate=PRED.hasSubscription, id_only=False) for assoc in assocs: rr.delete_association(assoc) try: pubsub.deactivate_subscription(assoc.o) except: pass pubsub.delete_subscription(assoc.o) def create_ingest_config(self): self.queue = IngestionQueue(name="test", type="testdata") # Create the ingestion config ingestion_config_id = self.ingestion_management.create_ingestion_configuration( name=self.ingest_name, exchange_point_id=self.exchange, queues=[self.queue] ) return ingestion_config_id def test_ingestion_config_crud(self): ingestion_config_id = self.create_ingest_config() ingestion_config = self.ingestion_management.read_ingestion_configuration(ingestion_config_id) self.assertTrue(ingestion_config.name == self.ingest_name) self.assertTrue(ingestion_config.queues[0].name == "test") self.assertTrue(ingestion_config.queues[0].type == "testdata") ingestion_config.name = "another" self.ingestion_management.update_ingestion_configuration(ingestion_config) # Create an association just to make sure that it will delete them sub = Subscription() sub_id, _ = self.resource_registry.create(sub) assoc_id, _ = self.resource_registry.create_association( subject=ingestion_config_id, predicate=PRED.hasSubscription, object=sub_id ) self.ingestion_management.delete_ingestion_configuration(ingestion_config_id) with self.assertRaises(NotFound): self.resource_registry.read(assoc_id) def test_list_ingestion(self): # Create the ingest_config config_id = self.create_ingest_config() retval = self.ingestion_management.list_ingestion_configurations(id_only=True) # Nice thing about this is that it breaks if r2dm adds an ingest_config self.assertTrue(config_id in retval)
class TestDataProductVersions(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.client = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #@unittest.skip('not working') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}}) def test_createDataProductVersionSimple(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='test', parameter_dictionary_id=pdict_id) # test creating a new data product which will also create the initial/default version log.debug('Creating new data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id = self.client.create_data_product(dp_obj, ctd_stream_def_id) log.debug('new dp_id = %s', str(dp_id)) dpc_id = self.client.create_data_product_collection(data_product_id=dp_id, collection_name='firstCollection', collection_description='collection desc') #test that the links exist version_ids, _ = self.rrclient.find_objects(subject=dpc_id, predicate=PRED.hasVersion, id_only=True) log.debug('version_ids = %s', str(version_ids)) self.assertTrue(version_ids, 'Failed to connect the data product to the version collection.') self.assertTrue(version_ids[0] == dp_id, 'Failed to connect the data product to the version collection.') # test creating a subsequent data product version which will update the data product pointers dp2_obj = IonObject(RT.DataProduct, name='DP2', description='a second dp', temporal_domain = tdom, spatial_domain = sdom) dp2_id = self.client.create_data_product(dp2_obj, ctd_stream_def_id) log.debug('second dp_id = %s', dp2_id) log.debug("adding product %s to collection %s", dp2_id, dpc_id) self.client.add_data_product_version_to_collection(data_product_id=dp2_id, data_product_collection_id=dpc_id, version_name="second version", version_description="a second version created" ) #test that the links exist version_ids = self.RR2.find_data_product_ids_of_data_product_collection_using_has_version(dpc_id) self.assertEqual(2, len(version_ids)) recent_version_id = self.client.get_current_version(dpc_id) self.assertEquals(recent_version_id, dp2_id) base_version_id = self.client.get_base_version(dpc_id) self.assertEquals(base_version_id, dp_id) #--------------------------------------------------------------------------------------------- # Now check that we can subscribe to the stream for the data product version #--------------------------------------------------------------------------------------------- # Activating the data products contained in the versions held by the data product collection data_product_collection_obj = self.rrclient.read(dpc_id) version_list = data_product_collection_obj.version_list self.assertEquals(len(version_list), 2) for version in version_list: data_product_id = version.data_product_id self.client.activate_data_product_persistence(data_product_id) streams = self.RR2.find_streams_of_data_product_using_has_stream(data_product_id) self.assertNotEqual(0, len(streams)) for stream in streams: self.assertTrue(stream.persisted) log.debug("This satisfies L4-CI-DM-RQ-053: 'The dynamic data distribution services shall support multiple " "versions of a given data topic.' This is true because we have shown above that we can persist the " "streams associated to the data product versions, and therefore we can subscribe to them. In " "test_oms_launch2.py, we have tested that activated data products like the ones we have here can be " "used by data processes to gather streaming data and it all works together correctly. This therefore " "completes the demonstration of req L4-CI-DM-RQ-053.") #--------------------------------------------------------------------------------------------- # Now delete all the created stuff and look for possible problems in doing so #--------------------------------------------------------------------------------------------- log.debug("deleting all data productgs") self.client.delete_data_product(dp_id) self.client.delete_data_product(dp2_id) self.client.delete_data_product_collection(dpc_id) self.client.force_delete_data_product_collection(dpc_id) # now try to get the deleted dp object self.assertRaises(NotFound, self.client.read_data_product_collection, dpc_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() #self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources( RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def cleanupprocs(self): stm = os.popen('ps -e | grep ion.agents.port.logger_process') procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) stm = os.popen('ps -e | grep ion.agents.instrument.zmq_driver_process') procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) # stm = os.popen('rm /tmp/*.pid.txt') @unittest.skip( "Deprecated by IngestionManagement refactor, timeout on start inst agent?" ) def test_deploy_activate_full(self): # ensure no processes or pids are left around by agents or Sims #self.cleanupprocs() self.loggerpids = [] #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD) try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) #------------------------------- # Create Instrument Site #------------------------------- instrumentSite_obj = IonObject(RT.InstrumentSite, name='instrumentSite1', description="SBE37IMInstrumentSite") try: instrumentSite_id = self.omsclient.create_instrument_site( instrument_site=instrumentSite_obj, parent_id='') except BadRequest as ex: self.fail("failed to create new InstrumentSite: %s" % ex) print 'test_deployAsPrimaryDevice: new instrumentSite id = ', instrumentSite_id self.omsclient.assign_instrument_model_to_instrument_site( instModel_id, instrumentSite_id) #------------------------------- # Logical Transform: Output Data Products #------------------------------- # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Old InstrumentDevice #------------------------------- instDevice_obj = IonObject( RT.InstrumentDevice, name='SBE37IMDeviceYear1', description="SBE37IMDevice for the FIRST year of deployment", serial_number="12345") try: oldInstDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, oldInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print 'test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ', oldInstDevice_id self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='SiteDataProduct', description='SiteDataProduct', temporal_domain=tdom, spatial_domain=sdom) instrument_site_output_dp_id = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=instrument_site_output_dp_id) #self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects( instrument_site_output_dp_id, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects( instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='first deployment') oldDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id) #------------------------------- # Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information # cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear1', description="SBE37IMAgentInstanceYear1", port_agent_config=port_agent_config, stream_configurations=[raw_config, parsed_config]) oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, oldInstDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() #------------------------------- # Create CTD Parsed as the Year 1 data product and attach to instrument #------------------------------- print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year1', description='ctd stream test year 1', temporal_domain=tdom, spatial_domain=sdom) ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year1 self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects( ctd_parsed_data_product_year1, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams1 = ', stream_ids #------------------------------- # Create New InstrumentDevice #------------------------------- instDevice_obj_2 = IonObject( RT.InstrumentDevice, name='SBE37IMDeviceYear2', description="SBE37IMDevice for the SECOND year of deployment", serial_number="67890") try: newInstDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj_2) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, newInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print 'test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ', newInstDevice_id #set the LCSTATE self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE) instDevice_obj_2 = self.rrclient.read(newInstDevice_id) log.debug( "test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate)) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='second deployment') newDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id) #------------------------------- # Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4004, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 4005, 'data_port': 4006, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear2', description="SBE37IMAgentInstanceYear2", port_agent_config=port_agent_config) newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, newInstDevice_id) #------------------------------- # Create CTD Parsed as the Year 2 data product #------------------------------- dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year2', description='ctd stream test year 2', temporal_domain=tdom, spatial_domain=sdom) ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year2 self.damsclient.assign_data_product( input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects( ctd_parsed_data_product_year2, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug( "test_deployAsPrimaryDevice: create data process definition ctd_L0_all" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new ctd_L0_all data process definition: %s" % ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name='L0_Conductivity', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity') outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name='L0_Pressure', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure') outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name='L0_Temperature', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature') self.out_prod_dict = {} log.debug( "test_deployAsPrimaryDevice: create output data product L0 conductivity" ) ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_conductivity_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.out_prod_dict['conductivity'] = ctd_l0_conductivity_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug( "test_deployAsPrimaryDevice: create output data product L0 pressure" ) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_pressure_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.out_prod_dict['pressure'] = ctd_l0_pressure_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug( "test_deployAsPrimaryDevice: create output data product L0 temperature" ) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_temperature_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.out_prod_dict['temperature'] = ctd_l0_temperature_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product) #------------------------------- log.debug( "test_deployAsPrimaryDevice: create L0 all data_process start") try: out_data_products = self.out_prod_dict.values() ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], out_data_products) self.dataprocessclient.activate_data_process( ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "test_deployAsPrimaryDevice: create L0 all data_process return") #-------------------------------- # Activate the deployment #-------------------------------- self.omsclient.activate_deployment(oldDeployment_id) #------------------------------- # Launch InstrumentAgentInstance Sim1, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=oldInstAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=oldInstAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance( oldInstAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, oldInstDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance( oldInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent1_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim1 = ResourceAgentClient('iaclient Sim1', name=gate.process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim1 %s', self._ia_client_sim1 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1)) #------------------------------- # Launch InstrumentAgentInstance Sim2, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=newInstAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=newInstAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance( newInstAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, oldInstDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance( newInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent2_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim2 = ResourceAgentClient('iaclient Sim2', name=gate.process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim2 %s', self._ia_client_sim2 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2)) #------------------------------- # Streaming Sim1 (old instrument) #------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active %s", str(reply)) self.assertTrue(reply) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim1.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from START_AUTOSAMPLE: %s", str(retval)) #------------------------------- # Streaming Sim 2 (new instrument) #------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim2.execute_agent(cmd) log.debug( "test_deployAsPrimaryDevice: return value from go_active_sim2 %s", str(reply)) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim2.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s", str(retval)) gevent.sleep(10) #------------------------------- # Shutdown Sim1 (old instrument) #------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(5) #------------------------------- # Shutdown Sim2 (old instrument) #------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset_sim2 ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset_sim2 %s", str(reply)) time.sleep(5)
class UserNotificationIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.unsc = UserNotificationServiceClient(node=self.container.node) self.rrc = ResourceRegistryServiceClient(node=self.container.node) self.imc = IdentityManagementServiceClient(node=self.container.node) def xtest_find_event_types_for_resource(self): dataset_object = IonObject(RT.DataSet, name="dataset1") dataset_id, version = self.rrc.create(dataset_object) events = self.unsc.find_event_types_for_resource(dataset_id) log.debug("dataset events = " + str(events)) try: events = self.unsc.find_event_types_for_resource("bogus_id") self.fail("failed to detect non-existant resource") except: pass def test_create_two_user_notifications(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) self.unsc.create_notification(notification_object, user_id) notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['data']}) self.unsc.create_notification(notification_object, user_id) def test_delete_user_notifications(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object1 = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) notification1_id = self.unsc.create_notification(notification_object1, user_id) notification_object2 = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['data']}) notification2_id = self.unsc.create_notification(notification_object2, user_id) self.unsc.delete_notification(notification1_id) self.unsc.delete_notification(notification2_id) def test_find_user_notifications(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) self.unsc.create_notification(notification_object, user_id) notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['data']}) self.unsc.create_notification(notification_object, user_id) notifications = self.unsc.find_notifications_by_user(user_id) for n in notifications: log.debug("n = " +str(n)) def test_update_user_notification(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) notification_id = self.unsc.create_notification(notification_object, user_id) notification = self.rrc.read(notification_id) notification.origin_list = ['Some_Resource_Agent_ID5'] self.unsc.update_notification(notification) def test_send_notification_emails(self): user_identty_object = IonObject(RT.UserIdentity, name="user1") user_id = self.imc.create_user_identity(user_identty_object) user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}}) self.imc.create_user_info(user_id, user_info_object) notification_object = IonObject(RT.NotificationRequest, {"name":"notification1", "origin_list":['Some_Resource_Agent_ID1'], "events_list":['resource_lifecycle']}) self.unsc.create_notification(notification_object, user_id) notification_object = IonObject(RT.NotificationRequest, {"name":"notification2", "origin_list":['Some_Resource_Agent_ID2'], "events_list":['data']}) self.unsc.create_notification(notification_object, user_id) rle_publisher = ResourceLifecycleEventPublisher() rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event") de_publisher = DataEventPublisher() de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event") gevent.sleep(1) def test_find_events(self): rle_publisher = ResourceLifecycleEventPublisher(event_repo=self.container.event_repository) rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event1") rle_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event2") de_publisher = DataEventPublisher(event_repo=self.container.event_repository) de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event1") de_publisher.create_and_publish_event(origin='Some_Resource_Agent_ID2', description="DE test event2") events = self.unsc.find_events(origin='Some_Resource_Agent_ID1') for event in events: log.debug("event=" + str(event)) events = self.unsc.find_events(type='DataEvent') for event in events: log.debug("event=" + str(event))
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() # self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.container.start_rel_from_url("res/deploy/r2deploy.yml") print "started services" # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=""): # logger process producer_definition = ProcessDefinition(name=name + "_logger") producer_definition.executable = { "module": "ion.processes.data.stream_granule_logger", "class": "StreamGranuleLogger", } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = {"process": {"stream_id": stream_id}} pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration ) return pid def cleanupprocs(self): stm = os.popen("ps -e | grep ion.agents.port.logger_process") procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) stm = os.popen("ps -e | grep ion.agents.instrument.zmq_driver_process") procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) # stm = os.popen('rm /tmp/*.pid.txt') @unittest.skip("Deprecated by IngestionManagement refactor, timeout on start inst agent?") def test_deploy_activate_full(self): # ensure no processes or pids are left around by agents or Sims # self.cleanupprocs() self.loggerpids = [] # ------------------------------- # Create InstrumentModel # ------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) # ------------------------------- # Create InstrumentAgent # ------------------------------- instAgent_obj = IonObject( RT.InstrumentAgent, name="agent007", description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.debug("new InstrumentAgent id = %s", instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # ------------------------------- # Create Instrument Site # ------------------------------- instrumentSite_obj = IonObject(RT.InstrumentSite, name="instrumentSite1", description="SBE37IMInstrumentSite") try: instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id="") except BadRequest as ex: self.fail("failed to create new InstrumentSite: %s" % ex) print "test_deployAsPrimaryDevice: new instrumentSite id = ", instrumentSite_id self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id) # ------------------------------- # Logical Transform: Output Data Products # ------------------------------- # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( "ctd_parsed_param_dict", id_only=True ) parsed_stream_def_id = self.pubsubclient.create_stream_definition( name="parsed", parameter_dictionary_id=parsed_pdict_id ) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_raw_param_dict", id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition(name="raw", parameter_dictionary_id=raw_pdict_id) # ------------------------------- # Create Old InstrumentDevice # ------------------------------- instDevice_obj = IonObject( RT.InstrumentDevice, name="SBE37IMDeviceYear1", description="SBE37IMDevice for the FIRST year of deployment", serial_number="12345", ) try: oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print "test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ", oldInstDevice_id self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE) # ------------------------------- # Create Raw and Parsed Data Products for the device # ------------------------------- dp_obj = IonObject( RT.DataProduct, name="SiteDataProduct", description="SiteDataProduct", temporal_domain=tdom, spatial_domain=sdom, ) instrument_site_output_dp_id = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id ) self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=instrument_site_output_dp_id ) # self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasStream, None, True) log.debug("Data product streams1 = %s", stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True) log.debug("Data set for data_product_id1 = %s", dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger("ctd_parsed", stream_ids[0]) self.loggerpids.append(pid) self.omsclient.create_site_data_product(instrumentSite_id, instrument_site_output_dp_id) # ------------------------------- # Create Old Deployment # ------------------------------- deployment_obj = IonObject(RT.Deployment, name="first deployment") oldDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id) # ------------------------------- # Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information # cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, # ------------------------------- port_agent_config = { "device_addr": CFG.device.sbe37.host, "device_port": CFG.device.sbe37.port, "process_type": PortAgentProcessType.UNIX, "binary_path": "port_agent", "port_agent_addr": "localhost", "command_port": CFG.device.sbe37.port_agent_cmd_port, "data_port": CFG.device.sbe37.port_agent_data_port, "log_level": 5, "type": PortAgentType.ETHERNET, } raw_config = StreamConfiguration( stream_name="raw", parameter_dictionary_name="ctd_raw_param_dict", records_per_granule=2, granule_publish_rate=5, ) parsed_config = StreamConfiguration( stream_name="parsed", parameter_dictionary_name="ctd_parsed_param_dict", records_per_granule=2, granule_publish_rate=5, ) instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstanceYear1", description="SBE37IMAgentInstanceYear1", comms_device_address="sbe37-simulator.oceanobservatories.org", comms_device_port=4001, port_agent_config=port_agent_config, stream_configurations=[raw_config, parsed_config], ) oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, oldInstDevice_id ) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() # ------------------------------- # Create CTD Parsed as the Year 1 data product and attach to instrument # ------------------------------- print "Creating new CDM data product with a stream definition" dp_obj = IonObject( RT.DataProduct, name="ctd_parsed_year1", description="ctd stream test year 1", temporal_domain=tdom, spatial_domain=sdom, ) ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id ) print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year1 self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1 ) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True) print "test_deployAsPrimaryDevice: Data product streams1 = ", stream_ids # ------------------------------- # Create New InstrumentDevice # ------------------------------- instDevice_obj_2 = IonObject( RT.InstrumentDevice, name="SBE37IMDeviceYear2", description="SBE37IMDevice for the SECOND year of deployment", serial_number="67890", ) try: newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print "test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ", newInstDevice_id # set the LCSTATE self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE) instDevice_obj_2 = self.rrclient.read(newInstDevice_id) log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate)) # ------------------------------- # Create Old Deployment # ------------------------------- deployment_obj = IonObject(RT.Deployment, name="second deployment") newDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id) # ------------------------------- # Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information # ------------------------------- port_agent_config = { "device_addr": "sbe37-simulator.oceanobservatories.org", "device_port": 4004, "process_type": PortAgentProcessType.UNIX, "binary_path": "port_agent", "port_agent_addr": "localhost", "command_port": 4005, "data_port": 4006, "log_level": 5, "type": PortAgentType.ETHERNET, } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstanceYear2", description="SBE37IMAgentInstanceYear2", comms_device_address="sbe37-simulator.oceanobservatories.org", comms_device_port=4004, port_agent_config=port_agent_config, ) newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, newInstDevice_id ) # ------------------------------- # Create CTD Parsed as the Year 2 data product # ------------------------------- dp_obj = IonObject( RT.DataProduct, name="ctd_parsed_year2", description="ctd stream test year 2", temporal_domain=tdom, spatial_domain=sdom, ) ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id ) print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year2 self.damsclient.assign_data_product( input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2 ) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True) print "test_deployAsPrimaryDevice: Data product streams2 = ", stream_ids # ------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition # ------------------------------- log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all") dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L0_all", description="transform ctd package into three separate L0 streams", module="ion.processes.data.transforms.ctd.ctd_L0_all", class_name="ctd_L0_all", ) try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" % ex) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products # ------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name="L0_Conductivity", parameter_dictionary_id=parsed_pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding="conductivity" ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name="L0_Pressure", parameter_dictionary_id=parsed_pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding="pressure" ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name="L0_Temperature", parameter_dictionary_id=parsed_pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding="temperature" ) self.output_products = {} log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name="L0_Conductivity", description="transform output conductivity", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_conductivity_output_dp_obj, stream_definition_id=parsed_stream_def_id ) self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name="L0_Pressure", description="transform output pressure", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_pressure_output_dp_obj, stream_definition_id=parsed_stream_def_id ) self.output_products["pressure"] = ctd_l0_pressure_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name="L0_Temperature", description="transform output temperature", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_temperature_output_dp_obj, stream_definition_id=parsed_stream_def_id ) self.output_products["temperature"] = ctd_l0_temperature_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product) # ------------------------------- log.debug("test_deployAsPrimaryDevice: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products ) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug("test_deployAsPrimaryDevice: create L0 all data_process return") # -------------------------------- # Activate the deployment # -------------------------------- self.omsclient.activate_deployment(oldDeployment_id) # ------------------------------- # Launch InstrumentAgentInstance Sim1, connect to the resource agent client # ------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id) self.addCleanup( self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=oldInstAgentInstance_id ) # wait for start instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) gate = ProcessStateGate( self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING ) self.assertTrue( gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id, ) inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent1_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim1 = ResourceAgentClient( "iaclient Sim1", name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess() ) print "activate_instrument: got _ia_client_sim1 %s", self._ia_client_sim1 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1)) # ------------------------------- # Launch InstrumentAgentInstance Sim2, connect to the resource agent client # ------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id) self.addCleanup( self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=newInstAgentInstance_id ) # wait for start instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) gate = ProcessStateGate( self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING ) self.assertTrue( gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id, ) inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent2_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim2 = ResourceAgentClient( "iaclient Sim2", name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess() ) print "activate_instrument: got _ia_client_sim2 %s", self._ia_client_sim2 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2)) # ------------------------------- # Streaming Sim1 (old instrument) # ------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active %s", str(reply)) self.assertTrue(reply) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim1.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE: %s", str(retval)) # ------------------------------- # Streaming Sim 2 (new instrument) # ------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active_sim2 %s", str(reply)) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim2.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s", str(retval)) gevent.sleep(10) # ------------------------------- # Shutdown Sim1 (old instrument) # ------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(5) # ------------------------------- # Shutdown Sim2 (old instrument) # ------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset_sim2 ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset_sim2 %s", str(reply)) time.sleep(5)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 200.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = 100.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 50.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = 100.0 #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 150.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 300.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = 200.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 150.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = 200.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 250.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) self.get_datastore(dataset_ids[0]) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug("The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'") data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name,'DP1') self.assertEquals(data_product_object.description,'some new dp') log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @unittest.skip('OBE') def test_get_last_update(self): # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory) sdom = sdom.dump() tdom = tdom.dump() #@TODO: DO NOT DO THIS, WHEN THIS TEST IS REWRITTEN GET RID OF THIS, IT WILL FAIL, thanks -Luke parameter_dictionary = get_param_dict('ctd_parsed_param_dict') parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) data_product_id = self.dpsc_cli.create_data_product(data_product=dp_obj, stream_definition_id=self.stream_def_id, parameter_dictionary=parameter_dictionary) stream_ids, garbage = self.rrclient.find_objects(data_product_id, PRED.hasStream, id_only=True) stream_id = stream_ids[0] fake_lu = LastUpdate() fake_lu_doc = self.db._ion_object_to_persistence_dict(fake_lu) self.db.create_doc(fake_lu_doc, object_id=stream_id) #------------------------------------------ # Now execute #------------------------------------------ res = self.dpsc_cli.get_last_update(data_product_id=data_product_id) self.assertTrue(isinstance(res[stream_id], LastUpdate), 'retrieving documents failed') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ log.debug('test_createDataProduct: Creating new data product w/o a stream definition (L4-CI-SA-RQ-308)') # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) log.debug('new dp_id = %s' % dp_id) log.debug("test_createDataProduct: Data product info from registry %s (L4-CI-SA-RQ-308)", str(dp_obj)) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object try: dp_obj = self.dpsc_cli.read_data_product(dp_id) except NotFound as ex: pass else: self.fail("force deleted data product was found during read") # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) # now try to get the deleted dp object #todo: the RR should perhaps not return retired data products # dp_obj = self.dpsc_cli.read_data_product(dp_id) # now try to delete the already deleted dp object # log.debug( "deleting non-existing data product") # self.dpsc_cli.delete_data_product(dp_id) # Shut down container #container.stop() def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ log.debug('test_createDataProduct: Creating new data product w/o a stream definition (L4-CI-SA-RQ-308)') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) log.debug('new dp_id = %s' % dp_id) log.debug("test_createDataProduct: Data product info from registry %s (L4-CI-SA-RQ-308)", str(dp_obj)) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) self.get_datastore(dataset_ids[0]) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) # pid = self.container.spawn_process(name='dummy_process_for_test', # module='pyon.ion.process', # cls='SimpleProcess', # config={}) # dummy_process = self.container.proc_manager.procs[pid] self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object try: dp_obj = self.rrclient.read(dp_id) except NotFound as ex: pass else: self.fail("force_deleted data product was found during read")
class ProcessDispatcherServiceIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2cei.yml') self.rr_cli = ResourceRegistryServiceClient() self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node) self.process_definition = ProcessDefinition(name='test_process') self.process_definition.executable = {'module': 'ion.services.cei.test.test_process_dispatcher', 'class': 'TestProcess'} self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition) self.waiter = ProcessStateWaiter() def tearDown(self): self.waiter.stop() def test_create_schedule_cancel(self): process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) # verifies L4-CI-CEI-RQ141 and L4-CI-CEI-RQ142 self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, {}) self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING) # make sure process is readable directly from RR (mirrored) # verifies L4-CI-CEI-RQ63 # verifies L4-CI-CEI-RQ64 proc = self.rr_cli.read(pid) self.assertEqual(proc.process_id, pid) # now try communicating with the process to make sure it is really running test_client = TestClient() for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # verifies L4-CI-CEI-RQ147 # kill the process and start it again self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) self.waiter.stop() oldpid = pid pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.assertNotEqual(oldpid, pid) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # kill the process for good self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) def test_schedule_with_config(self): process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) # verifies L4-CI-CEI-RQ66 # feed in a string that the process will return -- verifies that # configuration actually makes it to the instantiated process test_response = uuid.uuid4().hex configuration = {"test_response" : test_response} pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration=configuration, process_id=pid) self.assertEqual(pid, pid2) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) test_client = TestClient() # verifies L4-CI-CEI-RQ139 # assure that configuration block (which can contain inputs, outputs, # and arbitrary config) 1) makes it to the process and 2) is returned # in process queries self.assertEqual(test_client.query(), test_response) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, configuration) # kill the process for good self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) def test_schedule_bad_config(self): process_schedule = ProcessSchedule() # a non-JSON-serializable IonObject o = ProcessTarget() with self.assertRaises(BadRequest) as ar: self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={"bad": o}) self.assertTrue(ar.exception.message.startswith("bad configuration")) def test_create_invalid_definition(self): # create process definition missing module and class # verifies L4-CI-CEI-RQ137 executable = dict(url="http://somewhere.com/something.py") definition = ProcessDefinition(name="test_process", executable=executable) with self.assertRaises(BadRequest) as ar: self.pd_cli.create_process_definition(definition)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Lifecycle self.assertEquals(len(extended_instrument.lcstate_transitions), 7) self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce'])) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) self.assertEquals(extended_platform.platform_agent._id, platform_agent_id) self.assertEquals(len(extended_platform.lcstate_transitions), 7) self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce'])) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(StatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config(self): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) iconfig_builder = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = {'lvl1': {'lvl2': 'lvl3val'}} required_config_keys = [ 'org_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alert_config', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'}, 'alerts': generic_alerts_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config, 'alerts': generic_alerts_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure() platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj) child_config = pconfig_builder.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure() platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id) child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ self.stream_def_id = self.pubsubcli.create_stream_definition( name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition( name='ingestion worker') ingestion_worker_definition.executable = { 'module': 'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class': 'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition( process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process( self.process_definitions['ingestion_worker'], configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @attr('EXT') @attr('PREP') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) # Assert that the data product has an associated stream def at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True) self.assertNotEquals(len(stream_ids), 0) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp') dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) group_names = self.dpsc_cli.get_data_product_group_list() self.assertIn("PRODNAME", group_names) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description, 'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual( ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual( 0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) def ion_object_encoder(obj): return obj.__dict__ #test prepare for create data_product_data = self.dpsc_cli.prepare_data_product_support() #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, "") self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual( len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual( len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual( len(data_product_data.associations['StreamDefinition']. associated_resources), 0) self.assertEqual( len(data_product_data.associations['Dataset'].associated_resources ), 0) #test prepare for update data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, dp_id) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual( len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual( len(data_product_data.associations['Dataset'].resources), 1) self.assertEqual( len(data_product_data.associations['StreamDefinition']. associated_resources), 1) self.assertEqual( data_product_data.associations['StreamDefinition']. associated_resources[0].s, dp_id) self.assertEqual( len(data_product_data.associations['Dataset'].associated_resources ), 1) self.assertEqual( data_product_data.associations['Dataset'].associated_resources[0]. s, dp_id) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) # Assert that there are no associated streams leftover after deleting the data product stream_ids, assoc_ids = self.rrclient.find_objects( dp_id, PRED.hasStream, RT.Stream, True) self.assertEquals(len(stream_ids), 0) self.assertEquals(len(assoc_ids), 0) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_id = self.dpsc_cli.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_derived_data_product(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id) dp = DataProduct(name='Instrument DP') dp_id = self.dpsc_cli.create_data_product( dp, stream_definition_id=ctd_stream_def_id) self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id) self.dpsc_cli.activate_data_product_persistence(dp_id) self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Make the derived data product simple_stream_def_id = self.pubsubcli.create_stream_definition( name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time', 'temp']) tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED) tempwat_dp_id = self.dpsc_cli.create_data_product( tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id) self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) rdt['pressure'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id, route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True) tempwat_dataset_id = tempwat_dataset_ids[0] granule = self.data_retriever.retrieve( tempwat_dataset_id, delivery_format=simple_stream_def_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) self.assertEquals(set(rdt.fields), set(['time', 'temp'])) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product=dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # Subscribe to persist events #------------------------------------------------------------------------------------------------ queue = gevent.queue.Queue() def info_event_received(message, headers): queue.put(message) es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True) es.start() self.addCleanup(es.stop) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id, route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug( "The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'" ) data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name, 'DP1') self.assertEquals(data_product_object.description, 'some new dp') log.debug( "Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description, data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) dataset_modified.clear() rdt['time'] = np.arange(20, 40) publisher.publish(rdt.to_granule()) self.assertFalse(dataset_modified.wait(2)) self.dpsc_cli.activate_data_product_persistence(dp_id) dataset_modified.clear() publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_almost_equal(rdt['time'], np.arange(40)) dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True) self.assertEquals(len(dataset_ids), 1) self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id) info_event_counter = 0 runtime = 0 starttime = time.time() caught_events = [] #check that the four InfoStatusEvents were received while info_event_counter < 4 and runtime < 60: a = queue.get(timeout=60) caught_events.append(a) info_event_counter += 1 runtime = time.time() - starttime self.assertEquals(info_event_counter, 4)
class CatalogManagementIntTest(IonIntegrationTestCase): def setUp(self): super(CatalogManagementIntTest,self).setUp() self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.cms_cli = CatalogManagementServiceClient() self.rr_cli = ResourceRegistryServiceClient() def test_create_catalog(self): #------------------------------------- # Tests basic catalog creation #------------------------------------- index1 = Index(name='index1') index1.options.attribute_match = ['common'] index_id, _ = self.rr_cli.create(index1) catalog_id = self.cms_cli.create_catalog('common_catalog', ['common']) self.assertTrue(index_id in self.cms_cli.list_indexes(catalog_id)) def test_create_catalog_matching(self): #------------------------------------------- # Tests catalog creation and index matching #------------------------------------------- indexes = [ Index(name='index1'), Index(name='index2'), Index(name='index3') ] indexes[0].options = SearchOptions(**{ # Devices 'attribute_match' : ['name','model','serial'], 'geo_fields' : ['nominal_location'] }) indexes[1].options = SearchOptions(**{ # BankAccount 'attribute_match' : ['name','description','owner'], 'range_fields' : ['cash_balance'] }) indexes[2].options = SearchOptions(**{ # Hybrid 'attribute_match' : ['name'], 'range_fields' : ['arc'], 'geo_fields' : ['nominal_location'] }) index_ids = list([ self.rr_cli.create(i)[0] for i in indexes]) devices_catalog_id = self.cms_cli.create_catalog('devices_catalog', ['model']) self.assertTrue(index_ids[0] in self.cms_cli.list_indexes(devices_catalog_id), 'The catalog did\'nt match the correct index.') accounts_catalog_id = self.cms_cli.create_catalog('accounts_catalog', ['cash_balance']) self.assertTrue(index_ids[1] in self.cms_cli.list_indexes(accounts_catalog_id), 'The catalog did\'nt match the correct index.') geo_catalog_id = self.cms_cli.create_catalog('geo_catalog', ['nominal_location']) self.assertTrue(index_ids[2] in self.cms_cli.list_indexes(geo_catalog_id), 'The catalog did\'nt match the correct index.') names_catalog_id = self.cms_cli.create_catalog('names_catalog', ['name']) names_list = self.cms_cli.list_indexes(names_catalog_id) self.assertTrue(set(index_ids) == set(names_list), 'The catalog did\'nt match the correct index.') def test_catalog_field_exclusion(self): half_match = Index(name='half_match', options=SearchOptions(attribute_match=['one'])) full_match = Index(name='full_match', options=SearchOptions(attribute_match=['one','two'])) half_match_id, _ = self.rr_cli.create(half_match) full_match_id, _ = self.rr_cli.create(full_match) catalog_id = self.cms_cli.create_catalog('test_cat', keywords=['one','two']) index_list = self.cms_cli.list_indexes(catalog_id, id_only=True) self.assertTrue(index_list == [full_match_id]) def test_update_catalog(self): empty_catalog_id = self.cms_cli.create_catalog('empty_catalog') empty_catalog = self.rr_cli.read(empty_catalog_id) empty_catalog.description = 'testing update' self.cms_cli.update_catalog(empty_catalog) empty_catalog = self.rr_cli.read(empty_catalog_id) self.assertTrue(empty_catalog.description == 'testing update') def test_read_catalog(self): catalog_res = Catalog(name='fake_catalog') catalog_id, _ = self.rr_cli.create(catalog_res) cat = self.cms_cli.read_catalog(catalog_id) self.assertTrue(cat.name == 'fake_catalog') def test_delete_catalog(self): empty_catalog_id, _ = self.rr_cli.create(Catalog(name='empty_catalog')) self.rr_cli.read(empty_catalog_id) # Throws exception if it doesn't exist self.cms_cli.delete_catalog(empty_catalog_id) with self.assertRaises(NotFound): self.rr_cli.read(empty_catalog_id) with self.assertRaises(NotFound): self.cms_cli.read_catalog(empty_catalog_id)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node) def test_createDataProcess(self): #--------------------------------------------------------------------------- # Data Process Definition #--------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # Make assertion on the newly registered data process definition data_process_definition = self.rrclient.read(dprocdef_id) self.assertEquals(data_process_definition.name, 'ctd_L0_all') self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams') self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all') self.assertEquals(data_process_definition.class_name, 'ctd_L0_all') # Read the data process definition using data process management and make assertions dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id) self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all') self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all') #--------------------------------------------------------------------------- # Create an input instrument #--------------------------------------------------------------------------- instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) # Register the instrument so that the data producer and stream object are created data_producer_id = self.damsclient.register_instrument(instrument_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id ) # Assert that the link between the stream definition and the data process definition was done assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True) self.assertIsNotNone(assocs) #--------------------------------------------------------------------------- # Input Data Product #--------------------------------------------------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() input_dp_obj = IonObject( RT.DataProduct, name='InputDataProduct', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test') #Make assertions on the input data product created input_dp_obj = self.rrclient.read(input_dp_id) self.assertEquals(input_dp_obj.name, 'InputDataProduct') self.assertEquals(input_dp_obj.description, 'some new dp') self.damsclient.assign_data_product(instrument_id, input_dp_id) # Retrieve the stream via the DataProduct->Stream associations stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True) self.in_stream_id = stream_ids[0] #--------------------------------------------------------------------------- # Output Data Product #--------------------------------------------------------------------------- outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' ) outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' ) outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' ) self.output_products={} output_dp_obj = IonObject(RT.DataProduct, name='conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id) self.output_products['conductivity'] = output_dp_id_1 output_dp_obj = IonObject(RT.DataProduct, name='pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id) self.output_products['pressure'] = output_dp_id_2 output_dp_obj = IonObject(RT.DataProduct, name='temperature', description='transform output ', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id) self.output_products['temperature'] = output_dp_id_3 #--------------------------------------------------------------------------- # Create the data process #--------------------------------------------------------------------------- def _create_data_process(): dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products) return dproc_id dproc_id = _create_data_process() # Make assertions on the data process created data_process = self.dataprocessclient.read_data_process(dproc_id) # Assert that the data process has a process id attached self.assertIsNotNone(data_process.process_id) # Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute self.assertIsNotNone(data_process.input_subscription_id) output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True) self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3])) @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}}) def test_createDataProcessUsingSim(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg") instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) #------------------------------- # Create CTD Raw as the second data product #------------------------------- raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj.name = 'ctd_raw' ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # Create listener for data process events and verify that events are received. #------------------------------- # todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events #todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_id = data_process.process_id self.addCleanup(self.process_dispatcher.cancel_process, process_id) #------------------------------- # Wait until the process launched in the create_data_process() method is actually running, before proceeding further in this test #------------------------------- gate = ProcessStateGate(self.process_dispatcher.read_process, process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The data process (%s) did not spawn in 30 seconds" % process_id) #------------------------------- # Retrieve a list of all data process defintions in RR and validate that the DPD is listed #------------------------------- # todo: Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions # todo: data topics are being handled by pub sub at the level of streams self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) #todo: check that activate event is received L4-CI-SA-RQ-367 #todo... (it looks like no event is being published when the data process is activated... so below, we just check for now # todo... that the subscription is indeed activated) (Swarbhanu) # todo: monitor process to see if it is active (sa-rq-182) ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id) input_subscription_id = ctd_l0_all_data_process.input_subscription_id subs = self.rrclient.read(input_subscription_id) self.assertTrue(subs.activated) # todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the extended resources for data process definition and for data process #------------------------------- extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) self.assertEqual(1, len(extended_process_definition.data_processes)) log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition)) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEqual(1, len(extended_process.input_data_products)) log.debug("test_createDataProcess: extended_process %s", str(extended_process)) ################################ Test the removal of data processes ################################## #------------------------------------------------------------------- # Test the deleting of the data process #------------------------------------------------------------------- # Before deleting, get the input streams, output streams and the subscriptions so that they can be checked after deleting # dp_obj_1 = self.rrclient.read(ctd_l0_all_data_process_id) # input_subscription_id = dp_obj_1.input_subscription_id # out_prods, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) # in_prods, _ = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasInputProduct, id_only=True) # in_streams = [] # for in_prod in in_prods: # streams, _ = self.rrclient.find_objects(in_prod, PRED.hasStream, id_only=True) # in_streams.extend(streams) # out_streams = [] # for out_prod in out_prods: # streams, _ = self.rrclient.find_objects(out_prod, PRED.hasStream, id_only=True) # out_streams.extend(streams) # Deleting the data process self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id) # Check that the data process got removed. Check the lcs state. It should be retired dp_obj = self.rrclient.read(ctd_l0_all_data_process_id) self.assertEquals(dp_obj.lcstate, LCS.RETIRED) # Check for process defs still attached to the data process dpd_assn_ids = self.rrclient.find_associations(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcessDefinition, id_only=True) self.assertEquals(len(dpd_assn_ids), 0) # Check for output data product still attached to the data process out_products, assocs = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) self.assertEquals(len(out_products), 0) self.assertEquals(len(assocs), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check of the data process has been deactivated self.assertIsNone(dp_obj.input_subscription_id) # Read the original subscription id of the data process and check that it has been deactivated with self.assertRaises(NotFound): self.pubsubclient.read_subscription(input_subscription_id) #------------------------------------------------------------------- # Delete the data process definition #------------------------------------------------------------------- # before deleting, get the process definition being associated to in order to be able to check later if the latter gets deleted as it should proc_def_ids, proc_def_asocs = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasProcessDefinition) self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id) # check that the data process definition has been retired dp_proc_def = self.rrclient.read(ctd_L0_all_dprocdef_id) self.assertEquals(dp_proc_def.lcstate, LCS.RETIRED) # Check for old associations of this data process definition proc_defs, proc_def_asocs = self.rrclient.find_objects(ctd_L0_all_dprocdef_id, PRED.hasProcessDefinition) self.assertEquals(len(proc_defs), 0) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject= ctd_L0_all_dprocdef_id, id_only=True) self.assertEquals(len(obj_assns), 0) ################################ Test the removal of data processes ################################## # Try force delete... This should simply delete the associations and the data process object # from the resource registry #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id) #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process definition #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id) def test_transform_function_crd(self): tf = TransformFunction(name='simple', module='pyon.ion.process', cls='SimpleProcess') tf_id = self.dataprocessclient.create_transform_function(tf) self.assertTrue(tf_id) tf2_id = self.dataprocessclient.create_transform_function(tf) self.assertEquals(tf_id, tf2_id) tf_obj = self.dataprocessclient.read_transform_function(tf_id) self.assertEquals([tf.name, tf.module, tf.cls, tf.function_type], [tf_obj.name, tf_obj.module, tf_obj.cls, tf_obj.function_type]) tf.module = 'dev.null' self.assertRaises(BadRequest, self.dataprocessclient.create_transform_function, tf) self.dataprocessclient.delete_transform_function(tf_id) self.assertRaises(NotFound, self.dataprocessclient.read_transform_function, tf_id)
class TestResourceRegistry(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2coi.yml') # Now create client to bank service self.resource_registry_service = ResourceRegistryServiceClient( node=self.container.node) def test_crud(self): # Some quick registry tests # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", name="name", foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name", "foo": "bar"}) self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name"}, foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Instantiate an object obj = IonObject("UserInfo", name="name") # Can set attributes that aren't in the object's schema with self.assertRaises(AttributeError) as cm: setattr(obj, "foo", "bar") self.assertTrue( cm.exception.message == "'UserInfo' object has no attribute 'foo'") # Cam't call update with object that hasn't been persisted with self.assertRaises(BadRequest) as cm: self.resource_registry_service.update(obj) self.assertTrue( cm.exception.message.startswith( "Object does not have required '_id' or '_rev' attribute")) # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) # Cannot create object with _id and _rev fields pre-set with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create(read_obj) self.assertTrue( cm.exception.message.startswith("Doc must not have '_id'")) # Update object read_obj.name = "John Doe" self.resource_registry_service.update(read_obj) # Update should fail with revision mismatch with self.assertRaises(Conflict) as cm: self.resource_registry_service.update(read_obj) self.assertTrue( cm.exception.message.startswith( "Object not based on most current version")) # Re-read and update object read_obj = self.resource_registry_service.read(obj_id) self.resource_registry_service.update(read_obj) # Delete object self.resource_registry_service.delete(obj_id) # Make sure read, update and delete report error with self.assertRaises(NotFound) as cm: self.resource_registry_service.read(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.update(read_obj) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) # Owner creation tests user = IonObject("ActorIdentity", name='user') uid, _ = self.resource_registry_service.create(user) inst = IonObject("InstrumentDevice", name='instrument') iid, _ = self.resource_registry_service.create( inst, headers={'ion-actor-id': str(uid)}) ids, _ = self.resource_registry_service.find_objects(iid, PRED.hasOwner, RT.ActorIdentity, id_only=True) self.assertEquals(len(ids), 1) assoc = self.resource_registry_service.read(ids[0]) self.resource_registry_service.delete(iid) with self.assertRaises(NotFound) as ex: assoc = self.resource_registry_service.read(iid) def test_lifecycle(self): att = IonObject("InstrumentDevice", name='mine', description='desc') rid, rev = self.resource_registry_service.create(att) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.name, att.name) self.assertEquals(att1.lcstate, LCS.DRAFT_PRIVATE) new_state = self.resource_registry_service.execute_lifecycle_transition( rid, LCE.PLAN) self.assertEquals(new_state, LCS.PLANNED_PRIVATE) att2 = self.resource_registry_service.read(rid) self.assertEquals(att2.lcstate, LCS.PLANNED_PRIVATE) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.execute_lifecycle_transition( rid, LCE.UNANNOUNCE) self.assertTrue( "type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce" in cm.exception.message) new_state = self.resource_registry_service.execute_lifecycle_transition( rid, LCE.DEVELOP) self.assertEquals(new_state, LCS.DEVELOPED_PRIVATE) self.assertRaises( iex.BadRequest, self.resource_registry_service.execute_lifecycle_transition, resource_id=rid, transition_event='NONE##') self.resource_registry_service.set_lifecycle_state( rid, LCS.INTEGRATED_PRIVATE) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.lcstate, LCS.INTEGRATED_PRIVATE) def test_association(self): # Instantiate ActorIdentity object actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create( actor_identity_obj) read_actor_identity_obj = self.resource_registry_service.read( actor_identity_obj_id) # Instantiate UserInfo object user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create( user_info_obj) read_user_info_obj = self.resource_registry_service.read( user_info_obj_id) # Test create failures with self.assertRaises(AttributeError) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.bogus, user_info_obj_id) self.assertTrue(cm.exception.message == "bogus") # Predicate not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, None, user_info_obj_id) self.assertTrue( cm.exception.message == "Association must have all elements set") # Bad association type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, 'bogustype') self.assertTrue( cm.exception.message == "Unsupported assoc_type: bogustype") # Subject id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( None, PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Association must have all elements set") # Object id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, None) self.assertTrue( cm.exception.message == "Association must have all elements set") # Bad subject id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association( "bogus", PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # Bad object id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, "bogus") self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # _id missing from subject with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj, PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Subject id or rev not available") # _id missing from object with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj) self.assertTrue( cm.exception.message == "Object id or rev not available") # Wrong subject type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( user_info_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal subject type UserInfo for predicate hasInfo") # Wrong object type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id) self.assertTrue( cm.exception.message == "Illegal object type ActorIdentity for predicate hasInfo") # Create two different association types between the same subject and predicate assoc_id1, assoc_rev1 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) # Read object, subject res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True) self.assertEquals(res_obj1, user_info_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id) self.assertEquals(res_obj2._id, actor_identity_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True) self.assertEquals(res_obj2, actor_identity_obj_id) # Create a similar association to a specific revision # TODO: This is not a supported case so far assoc_id2, assoc_rev2 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, "H2R") # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations( None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, id_only=False) ret3 = self.resource_registry_service.find_associations( predicate=PRED.hasInfo, id_only=False) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo, read_user_info_obj) ret2 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations( None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None, True) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, id_only=True) ret3 = self.resource_registry_service.find_associations( predicate=PRED.hasInfo, id_only=True) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(None, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations( actor_identity_obj_id, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations( None, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations( actor_identity_obj, None, user_info_obj_id) self.assertTrue( cm.exception.message == "Object id not available in subject") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations( actor_identity_obj_id, None, user_info_obj) self.assertTrue( cm.exception.message == "Object id not available in object") # Find subjects (good cases) subj_ret1 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) subj_ret2 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_subjects( None, PRED.hasInfo, user_info_obj_id, True) subj_ret4 = self.resource_registry_service.find_subjects( None, None, read_user_info_obj, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_subjects( None, PRED.hasInfo, user_info_obj_id, False) subj_ret6 = self.resource_registry_service.find_subjects( None, None, read_user_info_obj, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find subjects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(None, None, None) self.assertTrue(cm.exception.message == "Must provide object") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.bogus, user_info_obj_id, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_subjects( RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.hasInfo, user_info_obj, True) self.assertTrue( cm.exception.message == "Object id not available in object") # Find objects (good cases) subj_ret1 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True) subj_ret2 = self.resource_registry_service.find_objects( read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, None, True) subj_ret4 = self.resource_registry_service.find_objects( actor_identity_obj_id, None, None, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, None, False) subj_ret6 = self.resource_registry_service.find_objects( read_actor_identity_obj, None, None, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find objects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(None, None, None) self.assertTrue(cm.exception.message == "Must provide subject") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects( actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue( cm.exception.message == "Object id not available in subject") # Get association (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(None, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association( actor_identity_obj_id, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association( None, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association( actor_identity_obj, None, user_info_obj_id) self.assertTrue( cm.exception.message == "Object id not available in subject") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association( actor_identity_obj_id, None, user_info_obj) self.assertTrue( cm.exception.message == "Object id not available in object") # Delete one of the associations self.resource_registry_service.delete_association(assoc_id2) assoc = self.resource_registry_service.get_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(assoc._id == assoc_id1) # Delete (bad cases) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete_association("bogus") self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # Delete other association self.resource_registry_service.delete_association(assoc_id1) # Delete resources self.resource_registry_service.delete(actor_identity_obj_id) self.resource_registry_service.delete(user_info_obj_id) def test_find_resources(self): with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_resources( RT.UserInfo, LCS.DRAFT, "name", False) self.assertTrue( cm.exception.message == "find by name does not support lcstate") ret = self.resource_registry_service.find_resources( RT.UserInfo, None, "name", False) self.assertTrue(len(ret[0]) == 0) # Instantiate an object obj = IonObject("InstrumentDevice", name="name") # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) ret = self.resource_registry_service.find_resources( RT.InstrumentDevice, None, "name", False) self.assertTrue(len(ret[0]) == 1) self.assertTrue(ret[0][0]._id == read_obj._id) ret = self.resource_registry_service.find_resources( RT.InstrumentDevice, LCS.DRAFT, None, False) self.assertTrue(len(ret[0]) == 1) self.assertTrue(ret[0][0]._id == read_obj._id) def test_attach(self): binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82" # Owner creation tests instrument = IonObject("InstrumentDevice", name='instrument') iid, _ = self.resource_registry_service.create(instrument) att = Attachment(content=binary, attachment_type=AttachmentType.BLOB) aid1 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid1) self.assertEquals(binary, att1.content) import base64 att = Attachment(content=base64.encodestring(binary), attachment_type=AttachmentType.ASCII) aid2 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid2) self.assertEquals(binary, base64.decodestring(att1.content)) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid1, aid2]) att_ids = self.resource_registry_service.find_attachments( iid, id_only=True, descending=True) self.assertEquals(att_ids, [aid2, aid1]) att_ids = self.resource_registry_service.find_attachments( iid, id_only=True, descending=True, limit=1) self.assertEquals(att_ids, [aid2]) atts = self.resource_registry_service.find_attachments(iid, id_only=False, limit=1) self.assertEquals(atts[0].content, att1.content) self.resource_registry_service.delete_attachment(aid1) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid2]) self.resource_registry_service.delete_attachment(aid2) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, []) def test_read_mult(self): test_resource1_id, _ = self.resource_registry_service.create( Resource(name='test1')) test_resource2_id, _ = self.resource_registry_service.create( Resource(name='test2')) res_list = [test_resource1_id, test_resource2_id] objects = self.resource_registry_service.read_mult(res_list) for o in objects: self.assertIsInstance(o, Resource) self.assertTrue(o._id in res_list) def test_find_associations_mult(self): dp = DataProcess() transform = Transform() pd = ProcessDefinition() dp_id, _ = self.resource_registry_service.create(dp) transform_id, _ = self.resource_registry_service.create(transform) pd_id, _ = self.resource_registry_service.create(pd) self.resource_registry_service.create_association( subject=dp_id, object=transform_id, predicate=PRED.hasTransform) self.resource_registry_service.create_association( subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition) results, _ = self.resource_registry_service.find_associations_mult( subjects=[dp_id], id_only=True) self.assertTrue(results == [transform_id]) results, _ = self.resource_registry_service.find_associations_mult( subjects=[dp_id, transform_id], id_only=True) results.sort() correct = [transform_id, pd_id] correct.sort() self.assertTrue(results == correct)
class VisualizationIntegrationTestHelper(IonIntegrationTestCase): def create_ctd_input_stream_and_data_product(self, data_product_name='ctd_parsed'): cc = self.container assertions = self.assertTrue # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.workflowclient = WorkflowManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node) self.vis_client = VisualizationServiceClient(node=self.container.node) #------------------------------- # Create CTD Parsed as the initial data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data') log.debug('Creating new CDM data product with a stream definition') craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name=data_product_name, description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product_id = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug('new ctd_parsed_data_product_id = %s' % ctd_parsed_data_product_id) #Only ever need one device for testing purposes. instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='SBE37IMDevice') if instDevice_obj: instDevice_id = instDevice_obj[0]._id else: instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product_id) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0 ) ctd_stream_id = stream_ids[0] return ctd_stream_id, ctd_parsed_data_product_id def create_data_product(self, dp_name = "", dp_description = ""): craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() # this creates a ParameterDictionary object parameter_dictionary = parameter_dictionary.dump() # this returns a python dictionary data_prod_obj = IonObject(RT.DataProduct, name=dp_name, description=dp_description, temporal_domain = tdom, spatial_domain = sdom) data_prod_id = self.create_data_product(data_prod_obj, stream_definition_id, parameter_dictionary) return data_prod_id, data_prod_obj def start_simple_input_stream_process(self, ctd_stream_id): return self.start_input_stream_process(ctd_stream_id) def start_sinusoidal_input_stream_process(self, ctd_stream_id): return self.start_input_stream_process(ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher', 'SinusoidalCtdPublisher') def start_input_stream_process(self, ctd_stream_id, module = 'ion.processes.data.ctd_stream_publisher', class_name= 'SimpleCtdPublisher'): ### ### Start the process for producing the CTD data ### # process definition for the ctd simulator... producer_definition = ProcessDefinition() producer_definition.executable = { 'module':module, 'class':class_name } ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(process_definition=producer_definition) # Start the ctd simulator to produce some data configuration = { 'process':{ 'stream_id':ctd_stream_id, } } ctd_sim_pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration) return ctd_sim_pid def start_output_stream_and_listen(self, ctd_stream_id, data_product_stream_ids, message_count_per_stream=10): cc = self.container assertions = self.assertTrue ### ### Make a subscriber in the test to listen for transformed data ### salinity_subscription_id = self.pubsubclient.create_subscription( query=StreamQuery(data_product_stream_ids), exchange_name = 'workflow_test', exchange_point = 'science_data', name = "test workflow transformations", ) pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, container=cc) result = gevent.event.AsyncResult() results = [] message_count = len(data_product_stream_ids) * message_count_per_stream def message_received(message, headers): # Heads results.append(message) if len(results) >= message_count: #Only wait for so many messages - per stream result.set(True) subscriber = subscriber_registrar.create_subscriber(exchange_name='workflow_test', callback=message_received) subscriber.start() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id) #Start the input stream process if ctd_stream_id is not None: ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) # Assert that we have received data assertions(result.get(timeout=30)) # stop the flow parse the messages... if ctd_stream_id is not None: self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id) subscriber.stop() return results def validate_messages(self, results): cc = self.container assertions = self.assertTrue first_salinity_values = None for message in results: rdt = RecordDictionaryTool.load_from_granule(message) try: temp = get_safe(rdt, 'temp') # psd = PointSupplementStreamParser(stream_definition=self.ctd_stream_def, stream_granule=message) # temp = psd.get_values('temperature') # log.info(psd.list_field_names()) except KeyError as ke: temp = None if temp is not None: assertions(isinstance(temp, numpy.ndarray)) log.info( 'temperature=' + str(numpy.nanmin(temp))) first_salinity_values = None else: #psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message) #log.info( psd.list_field_names()) # Test the handy info method for the names of fields in the stream def #assertions('salinity' in psd.list_field_names()) # you have to know the name of the coverage in stream def salinity = get_safe(rdt, 'salinity') #salinity = psd.get_values('salinity') log.info( 'salinity=' + str(numpy.nanmin(salinity))) # Check to see if salinity has values assertions(salinity != None) assertions(isinstance(salinity, numpy.ndarray)) assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0 if first_salinity_values is None: first_salinity_values = salinity.tolist() else: second_salinity_values = salinity.tolist() assertions(len(first_salinity_values) == len(second_salinity_values)) for idx in range(0,len(first_salinity_values)): assertions(first_salinity_values[idx]*2.0 == second_salinity_values[idx]) def validate_data_ingest_retrieve(self, dataset_id): assertions = self.assertTrue self.data_retriever = DataRetrieverServiceClient(node=self.container.node) #validate that data was ingested replay_granule = self.data_retriever.retrieve_last_granule(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) salinity = get_safe(rdt, 'salinity') assertions(salinity != None) #retrieve all the granules from the database and check the values replay_granule_all = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule_all) for k, v in rdt.iteritems(): if k == 'salinity': for val in numpy.nditer(v): assertions(val > 0) def create_salinity_data_process_definition(self): # Salinity: Data Process Definition #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='ctd_salinity') if len(dpd) > 0: return dpd[0] log.debug("Create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_salinity', description='create a salinity data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Excpetion as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) # create a stream definition for the data from the salinity Transform sal_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='Salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(sal_stream_def_id, ctd_L2_salinity_dprocdef_id ) return ctd_L2_salinity_dprocdef_id def create_salinity_doubler_data_process_definition(self): #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler') if len(dpd) > 0: return dpd[0] # Salinity Doubler: Data Process Definition log.debug("Create data process definition SalinityDoublerTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='salinity_doubler', description='create a salinity doubler data product', module='ion.processes.data.transforms.example_double_salinity', class_name='SalinityDoubler', process_source='SalinityDoubler source code here...') try: salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Exception as ex: self.fail("failed to create new SalinityDoubler data process definition: %s" %ex) # create a stream definition for the data from the salinity Transform salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler') self.dataprocessclient.assign_stream_definition_to_data_process_definition(salinity_double_stream_def_id, salinity_doubler_dprocdef_id ) return salinity_doubler_dprocdef_id def create_transform_process(self, data_process_definition_id, data_process_input_dp_id): data_process_definition = self.rrclient.read(data_process_definition_id) # Find the link between the output Stream Definition resource and the Data Process Definition resource stream_ids,_ = self.rrclient.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True) if not stream_ids: raise Inconsistent("The data process definition %s is missing an association to an output stream definition" % data_process_definition._id ) process_output_stream_def_id = stream_ids[0] #Concatenate the name of the workflow and data process definition for the name of the data product output data_process_name = data_process_definition.name # Create the output data product of the transform transform_dp_obj = IonObject(RT.DataProduct, name=data_process_name,description=data_process_definition.description) transform_dp_id = self.dataproductclient.create_data_product(transform_dp_obj, process_output_stream_def_id) self.dataproductclient.activate_data_product_persistence(data_product_id=transform_dp_id) #last one out of the for loop is the output product id output_data_product_id = transform_dp_id # Create the transform data process log.debug("create data_process and start it") data_process_id = self.dataprocessclient.create_data_process(data_process_definition._id, [data_process_input_dp_id], {'output':transform_dp_id}) self.dataprocessclient.activate_data_process(data_process_id) #Find the id of the output data stream stream_ids, _ = self.rrclient.find_objects(transform_dp_id, PRED.hasStream, None, True) if not stream_ids: raise Inconsistent("The data process %s is missing an association to an output stream" % data_process_id ) return data_process_id, output_data_product_id def create_google_dt_data_process_definition(self): #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform') if len(dpd) > 0: return dpd[0] # Data Process Definition log.debug("Create data process definition GoogleDtTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='google_dt_transform', description='Convert data streams to Google DataTables', module='ion.processes.data.transforms.viz.google_dt', class_name='VizTransformGoogleDT', process_source='VizTransformGoogleDT source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Exception as ex: self.fail("failed to create new VizTransformGoogleDT data process definition: %s" %ex) # create a stream definition for the data from the stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformGoogleDT.outgoing_stream_def, name='VizTransformGoogleDT') self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id ) return procdef_id def validate_google_dt_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) gdt_data = get_safe(rdt, 'google_dt_components') # IF this granule does not contains google dt, skip if gdt_data == None: continue gdt = gdt_data[0] assertions(gdt['viz_product_type'] == 'google_dt' ) assertions(len(gdt['data_description']) >= 0) # Need to come up with a better check assertions(len(gdt['data_content']) >= 0) def create_mpl_graphs_data_process_definition(self): #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform') if len(dpd) > 0: return dpd[0] #Data Process Definition log.debug("Create data process definition MatplotlibGraphsTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='mpl_graphs_transform', description='Convert data streams to Matplotlib graphs', module='ion.processes.data.transforms.viz.matplotlib_graphs', class_name='VizTransformMatplotlibGraphs', process_source='VizTransformMatplotlibGraphs source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Exception as ex: self.fail("failed to create new VizTransformMatplotlibGraphs data process definition: %s" %ex) # create a stream definition for the data stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformMatplotlibGraphs.outgoing_stream_def, name='VizTransformMatplotlibGraphs') self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id ) return procdef_id def validate_mpl_graphs_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) graphs = get_safe(rdt, 'matplotlib_graphs') if graphs == None: continue for graph in graphs[0]: # At this point only dictionaries containing image data should be passed # For some reason non dictionary values are filtering through. if not isinstance(graph, dict): continue assertions(graph['viz_product_type'] == 'matplotlib_graphs' ) # check to see if the list (numpy array) contains actual images assertions(imghdr.what(graph['image_name'], h = graph['image_obj']) == 'png') def validate_vis_service_google_dt_results(self, results): assertions = self.assertTrue assertions(results) gdt_str = (results.lstrip("google.visualization.Query.setResponse(")).rstrip(")") assertions(len(gdt_str) > 0) return def validate_vis_service_mpl_graphs_results(self, results): assertions = self.assertTrue assertions(results) # check to see if the object passed is a dictionary with a valid image object in it image_format = results["content_type"].lstrip("image/") assertions(imghdr.what(results['image_name'], h = base64.decodestring(results['image_obj'])) == image_format) return
class LastUpdateCacheTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.datastore_name = CACHE_DATASTORE_NAME self.container.start_rel_from_url('res/deploy/r2dm.yml') self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA) self.tms_cli = TransformManagementServiceClient() self.pubsub_cli = PubsubManagementServiceClient() self.pd_cli = ProcessDispatcherServiceClient() self.rr_cli = ResourceRegistryServiceClient() xs_dot_xp = CFG.core_xps.science_data try: self.XS, xp_base = xs_dot_xp.split('.') self.XP = '.'.join([bootstrap.get_sys_name(), xp_base]) except ValueError: raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp) def make_points(self,definition,stream_id='I am very special', N=100): from prototype.sci_data.constructor_apis import PointSupplementConstructor import numpy as np import random definition.stream_resource_id = stream_id total = N n = 10 # at most n records per granule i = 0 while i < total: r = random.randint(1,n) psc = PointSupplementConstructor(point_definition=definition, stream_id=stream_id) for x in xrange(r): i+=1 point_id = psc.add_point(time=i, location=(0,0,0)) psc.add_scalar_point_coverage(point_id=point_id, coverage_id='temperature', value=np.random.normal(loc=48.0,scale=4.0, size=1)[0]) psc.add_scalar_point_coverage(point_id=point_id, coverage_id='pressure', value=np.float32(1.0)) psc.add_scalar_point_coverage(point_id=point_id, coverage_id='conductivity', value=np.float32(2.0)) granule = psc.close_stream_granule() hdf_string = granule.identifiables[definition.data_stream_id].values granule.identifiables[definition.data_stream_id].values = hdf_string yield granule return def start_worker(self): proc_def = ProcessDefinition() proc_def.executable['module'] = 'ion.processes.data.last_update_cache' proc_def.executable['class'] = 'LastUpdateCache' proc_def_id = self.pd_cli.create_process_definition(process_definition=proc_def) subscription_id = self.pubsub_cli.create_subscription(query=ExchangeQuery(), exchange_name='ingestion_aggregate') config = { 'couch_storage' : { 'datastore_name' :self.datastore_name, 'datastore_profile' : 'SCIDATA' } } transform_id = self.tms_cli.create_transform( name='last_update_cache', description='LastUpdate that compiles an aggregate of metadata', in_subscription_id=subscription_id, process_definition_id=proc_def_id, configuration=config ) self.tms_cli.activate_transform(transform_id=transform_id) transform = self.rr_cli.read(transform_id) pid = transform.process_id handle = self.container.proc_manager.procs[pid] return handle @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_last_update_cache(self): handle = self.start_worker() queue = Queue() o_process = handle.process def new_process(msg): o_process(msg) queue.put(True) handle.process = new_process definition = SBE37_CDM_stream_definition() publisher = Publisher() stream_def_id = self.pubsub_cli.create_stream_definition(container=definition) stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id) time = float(0.0) for granule in self.make_points(definition=definition, stream_id=stream_id, N=10): publisher.publish(granule, to_name=(self.XP, stream_id+'.data')) # Determinism sucks try: queue.get(timeout=5) except Empty: self.assertTrue(False, 'Process never received the message.') doc = self.db.read(stream_id) ntime = doc.variables['time'].value self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.') time = ntime
class IndexManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.ims_cli = IndexManagementServiceClient() self.rr_cli = ResourceRegistryServiceClient() self.index_name = 'test_index' def test_create_elasticsearch_index(self): index_name = self.index_name ims_cli = self.ims_cli rr_cli = self.rr_cli options = SearchOptions() options.attribute_match = ['test_field'] index_id = ims_cli.create_index( name=index_name, content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=options ) index_result = self.rr_cli.read(index_id) self.assertIsInstance(index_result,ElasticSearchIndex) self.assertTrue(index_result.name == index_name) #====================================== # Clean up #====================================== rr_cli.delete(index_id) def test_create_couchdb_index(self): index_name = self.index_name ims_cli = self.ims_cli rr_cli = self.rr_cli options = SearchOptions() options.attribute_match = ['name'] index_id = ims_cli.create_index( index_name, content_type=IndexManagementService.COUCHDB_INDEX, options=options, datastore_name='fake', view_name='fake/by_fake' ) index_result = self.rr_cli.read(index_id) self.assertIsInstance(index_result,CouchDBIndex) self.assertTrue(index_result.name==index_name) #====================================== # Clean up #====================================== rr_cli.delete(index_id) def test_read_index(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name index_res = Index(name=index_name) index_id, _ = rr_cli.create(index_res) index = ims_cli.read_index(index_id) self.assertIsInstance(index,Index) self.assertTrue(index.name==index_name) rr_cli.delete(index_id) def test_delete_index(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name index_res = Index(name=index_name) index_id, _ = rr_cli.create(index_res) ims_cli.delete_index(index_id) with self.assertRaises(NotFound): rr_cli.delete(index_id) def test_update_index(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name index_res = Index(name=index_name) index_id, _ = rr_cli.create(index_res) index = ims_cli.read_index(index_id) index.name = 'another' ims_cli.update_index(index) index = rr_cli.read(index_id) self.assertTrue(index.name == 'another') def test_find_indexes(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name #====================================== # Index Pool #====================================== indexes = [ Index(name='first'), Index(name='second'), Index(name='third') ] id_pool = list() for index in indexes: id_pool.append(rr_cli.create(index)[0]) index_id = ims_cli.find_indexes(index_name='second') index = ims_cli.read_index(index_id) self.assertTrue(index.name=='second') #====================================== # Clean up #====================================== for index_id in id_pool: rr_cli.delete(index_id) def test_create_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli with self.assertRaises(BadRequest): ims_cli.create_collection('failing_collection') resources = [ Resource(), Resource(), Resource() ] resources = [ rr_cli.create(i)[0] for i in resources ] collection_id = ims_cli.create_collection('working_collection',resources) collection = rr_cli.read(collection_id) collection_resources = ims_cli.list_collection_resources(collection_id, id_only=True) self.assertTrue(set(collection_resources) == set(resources), '%s != %s' % (set(collection_resources) , set(resources))) def test_read_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli collection = Collection(name='working_collection') collection_id, _ = rr_cli.create(collection) collection = ims_cli.read_collection(collection_id) self.assertTrue(collection.name == 'working_collection') def test_update_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli collection = Collection(name='useful_collection') collection_id, _ = rr_cli.create(collection) collection = rr_cli.read(collection_id) collection.name = 'nub' ims_cli.update_collection(collection) collection = rr_cli.read(collection_id) self.assertTrue(collection.name=='nub') def test_delete_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli res = Resource() res_id, rev = rr_cli.create(res) collection_id = ims_cli.create_collection(name='test_collection', resources=[res_id]) ims_cli.delete_collection(collection_id) with self.assertRaises(NotFound): rr_cli.read(collection_id) def test_list_collection_resources(self): ims_cli = self.ims_cli rr_cli = self.rr_cli #======================================== # Resource Pool #======================================== resources = [ InformationResource(name='bean_counter'), InformationResource(name='lunar_rock'), InformationResource('aperature'), InformationResource('lemons') ] resources = [ rr_cli.create(i)[0] for i in resources ] collection = Collection(name='park_bench') collection_id = ims_cli.create_collection(name='park_bench', resources=resources) retval = ims_cli.list_collection_resources(collection_id, id_only=True) retval.sort() resources.sort() self.assertTrue(retval == resources, '%s != %s' %(retval , resources)) def test_find_collection(self): res_id, _ = self.rr_cli.create(Resource(name='test_res')) collection_id = self.ims_cli.create_collection('test', [res_id]) retval = self.ims_cli.find_collection(collection_name='test') self.assertTrue(retval[0] == collection_id) retval = self.ims_cli.find_collection(resource_ids=[res_id]) self.assertTrue(retval[0] == collection_id)
class TestAgentLaunchOps(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient( node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return def test_get_agent_client_noprocess(self): inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice)) iap = ResourceAgentClient._get_agent_process_id(inst_device_id) # should be no running agent self.assertIsNone(iap) # should raise NotFound self.assertRaises(NotFound, ResourceAgentClient, inst_device_id) def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.IMS.create_instrument_device( instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) spdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence( data_product_id=data_product_id1) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') data_product_id2 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence( data_product_id=data_product_id2) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance( instAgentInstance_id) gate = AgentProcessStateGate(self.PDC.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) if "BAD_DATA" == instance_obj.driver_config["comms_config"]: print "Saved config:" print snap_obj.content self.fail("Saved config was not properly restored") self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config_hasDevice(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(parent_device_id, PRED.hasDevice, child_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_objects(subject=parent_device_id, predicate=PRED.hasDevice, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasDevice") def test_agent_instance_config_hasNetworkParent(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(child_device_id, PRED.hasNetworkParent, parent_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_subjects(object=parent_device_id, predicate=PRED.hasNetworkParent, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasNetworkParent") def base_agent_instance_config( self, assign_child_platform_to_parent_platform_fn, find_child_platform_ids_of_parent_platform_fn): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC config_builder = DotDict config_builder.i = None config_builder.p = None def refresh_pconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.p = PlatformAgentConfigurationBuilder(clients) def refresh_iconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.i = InstrumentAgentConfigurationBuilder(clients) org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = [{'lvl2': 'lvl3val'}] required_config_keys = [ 'org_governance_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alerts_config', 'children' ] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = { 'process_type': ('ZMQPyClassDriverLauncher', ), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertIn('resource_id', config['agent']) self.assertEqual(device_id, config['agent']['resource_id']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) # TODO(OOIION-1495) review the asserts below related with # requiring 'ports' to be present in the driver_config. # See recent adjustment in agent_configuration_builder.py, # which I did to avoid other tests to fail. # The asserts below would make the following tests fail: # test_agent_instance_config_hasDevice # test_agent_instance_config_hasNetworkParent def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('resource_id', config['agent']) self.assertEqual(device_id, config['agent']['resource_id']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) """ self.assertIn('ports', config['driver_config']) """ self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher', ), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) """ if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) """ def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) """ self.assertIn('ports', config['driver_config']) """ self.assertEqual(('ZMQPyClassDriverLauncher', ), config['driver_config']['process_type']) self.assertIn('resource_id', config['agent']) self.assertEqual(parent_device_id, config['agent']['resource_id']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) """ if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) """ self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(name='', agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old( RT.PlatformAgentInstance, { 'driver_config': { 'foo': 'bar' }, 'alerts': generic_alerts_config }) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance( platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') platform_agent_obj = any_old( RT.PlatformAgent, {"stream_configurations": [raw_config]}) platform_agent_id = self.IMS.create_platform_agent( platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device( any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct) dp_id = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.PlatformSite, name='sitePlatform') site_id = self.OMS.create_platform_site(platform_site=site_obj) # find current deployment using time constraints current_time = int(calendar.timegm(time.gmtime())) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj = IonObject( OT.PlatformPort, reference_designator='GA01SUMO-FI003-09-CTDMO0999', port_type=PortTypeEnum.UPLINK, ip_address=0) deployment_obj = IonObject( RT.Deployment, name='TestPlatformDeployment_' + name, description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds], port_assignments={platform_device_id: platform_port_obj}) deploy_id = self.OMS.create_deployment( deployment=deployment_obj, site_id=site_id, device_id=platform_device_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance( platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition( platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource( platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old( RT.InstrumentAgentInstance, { "startup_config": inst_startup_config, 'alerts': generic_alerts_config }) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance( instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') instrument_agent_obj = any_old( RT.InstrumentAgent, {"stream_configurations": [raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent( instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device( any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct) dp_id = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product( input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.InstrumentSite, name='siteInstrument') site_id = self.OMS.create_instrument_site(instrument_site=site_obj) # find current deployment using time constraints current_time = int(calendar.timegm(time.gmtime())) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj = IonObject( OT.PlatformPort, reference_designator='GA01SUMO-FI003-08-CTDMO0888', port_type=PortTypeEnum.PAYLOAD, ip_address=0) deployment_obj = IonObject( RT.Deployment, name='TestDeployment for Cabled Instrument', description='some new deployment', context=IonObject(OT.CabledInstrumentDeploymentContext), constraint_list=[temporal_bounds], port_assignments={instrument_device_id: platform_port_obj}) deploy_id = self.OMS.create_deployment( deployment=deployment_obj, site_id=site_id, device_id=instrument_device_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance( instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition( instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource( instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug( "Testing that preparing a launcher without agent instance raises an error" ) refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated self.assertRaises(AssertionError, config_builder.p.prepare, will_launch=False) log.debug( "Making the structure for a platform agent, which will be the child" ) platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure( name='child') platform_agent_instance_child_obj = self.RR2.read( platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_child_obj) child_config = config_builder.p.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug( "Making the structure for a platform agent, which will be the parent" ) platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure( name='parent') platform_agent_instance_parent_obj = self.RR2.read( platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") assign_child_platform_to_parent_platform_fn(platform_device_child_id, platform_device_parent_id) child_device_ids = find_child_platform_ids_of_parent_platform_fn( platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure( ) instrument_agent_instance_obj = self.RR2.read( instrument_agent_instance_id) log.debug("Testing instrument config") refresh_iconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.i.set_agent_instance_object( instrument_agent_instance_obj) instrument_config = config_builder.i.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device_with_has_device( instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_platform_device_using_has_device( platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_parent_obj) full_config = config_builder.p.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) log.info("END base_agent_instance_config")
class TestIdentityManagementServiceInt(IonIntegrationTestCase): def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient(node=self.container.node) self.identity_management_service = IdentityManagementServiceClient(node=self.container.node) self.org_client = OrgManagementServiceClient(node=self.container.node) def test_actor_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity(actor_identity_obj) actor_identity = self.identity_management_service.read_actor_identity(user_id) actor_identity.name = 'Updated subject' self.identity_management_service.update_actor_identity(actor_identity) ai = self.identity_management_service.find_actor_identity_by_name(actor_identity.name) self._baseAssertEqual(ai.name, actor_identity.name) with self.assertRaises(NotFound): ai = self.identity_management_service.find_actor_identity_by_name("Yeah, well, you know, that's just, like, your opinion, man.") self._baseAssertEqual(ai.name, actor_identity.name) self.identity_management_service.delete_actor_identity(user_id) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) def test_user_credentials(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials(user_id, user_credentials_obj) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials("bad", self.subject) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials(user_id, "bad") self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials('bad', 'bad') self.assertTrue("does not exist" in cm.exception.message) self.identity_management_service.unregister_user_credentials(user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_user_info(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials(user_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info = self.identity_management_service.create_user_info(user_id, user_info_obj) with self.assertRaises(Conflict) as cm: self.identity_management_service.create_user_info(user_id, user_info_obj) self.assertTrue("UserInfo already exists for user id" in cm.exception.message) user_info_obj = self.identity_management_service.find_user_info_by_id(user_id) user_info_obj = self.identity_management_service.find_user_info_by_name("Foo") user_info_obj = self.identity_management_service.find_user_info_by_subject(self.subject) user_info_obj = self.identity_management_service.read_user_info(user_info) user_info_obj.name = 'Jane Doe' self.identity_management_service.update_user_info(user_info_obj) self.identity_management_service.delete_user_info(user_info) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_name("John Doe") self.assertEqual(cm.exception.message, 'UserInfo with name John Doe does not exist') with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_subject("Bogus subject") self.assertEqual(cm.exception.message, "UserCredentials with subject Bogus subject does not exist") self.identity_management_service.unregister_user_credentials(user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_signon(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" id, valid_until, registered = self.identity_management_service.signon(certificate, True) self.assertFalse(registered) id2, valid_until2, registered2 = self.identity_management_service.signon(certificate, True) self.assertFalse(registered2) self.assertTrue(id == id2) self.assertTrue(valid_until == valid_until2) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) self.identity_management_service.create_user_info(id, user_info_obj) id3, valid_until3, registered3 = self.identity_management_service.signon(certificate, True) self.assertTrue(registered3) self.assertTrue(id == id3) self.assertTrue(valid_until == valid_until3) @attr('EXT') def test_get_extended_user_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) actor_id = self.identity_management_service.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials(actor_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info_id = self.identity_management_service.create_user_info(actor_id, user_info_obj) ion_org = self.org_client.find_org() #Build the Service Agreement Proposal to to request a role but never close it sap = IonObject(OT.RequestRoleProposal,consumer=actor_id, provider=ion_org._id, role_name=ORG_MANAGER_ROLE ) sap_response = self.org_client.negotiate(sap) #Just grant the role anyway #self.org_client.grant_role(ion_org._id, actor_id, ORG_MANAGER_ROLE) with self.assertRaises(NotFound): self.identity_management_service.get_user_info_extension('That rug really tied the room together.') with self.assertRaises(BadRequest): self.identity_management_service.get_user_info_extension() #Check the user without the negotiation role request extended_user = self.identity_management_service.get_user_info_extension(user_info_id, org_id=ion_org._id) self.assertEqual(user_info_obj.type_,extended_user.resource.type_) self.assertEqual(len(extended_user.roles),1) self.assertEqual(len(extended_user.open_requests),1) self.assertEqual(extended_user.open_requests[0].org_id, ion_org._id) self.assertEqual(extended_user.open_requests[0].user_id, user_info_id) self.assertEqual(extended_user.open_requests[0].request_type, OT.RequestRoleProposal) self.assertEqual(len(extended_user.closed_requests),0) self.assertEqual(extended_user.open_requests[0]._id, extended_user.open_requests[0].negotiation_id) neg = self.resource_registry.read(object_id=extended_user.open_requests[0].negotiation_id) sap_response = Negotiation.create_counter_proposal(neg, ProposalStatusEnum.ACCEPTED, ProposalOriginatorEnum.PROVIDER) sap_response2 = self.org_client.negotiate(sap_response) #Now check the user after the negotiation has been accepted and the role granted extended_user = self.identity_management_service.get_user_info_extension(user_info_id, org_id=ion_org._id) self.assertEqual(user_info_obj.type_,extended_user.resource.type_) self.assertEqual(len(extended_user.roles),2) self.assertEqual(len(extended_user.open_requests),0) self.assertEqual(len(extended_user.closed_requests),1) self.assertEqual(extended_user.closed_requests[0].org_id, ion_org._id) self.assertEqual(extended_user.closed_requests[0].user_id, user_info_id) self.assertEqual(extended_user.closed_requests[0].request_type, OT.RequestRoleProposal) self.identity_management_service.delete_user_info(user_info_id) self.org_client.revoke_role(org_id=ion_org._id, actor_id=actor_id, role_name=ORG_MANAGER_ROLE) self.identity_management_service.unregister_user_credentials(actor_id, self.subject) self.identity_management_service.delete_actor_identity(actor_id) def test_account_merge(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" certificate_2 = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgIDAJ/lMA0GCSqGSIb3DQEBCwUAMGsxEzARBgoJkiaJk/IsZAEZFgNvcmcx FzAVBgoJkiaJk/IsZAEZFgdjaWxvZ29uMQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQ0lMb2dvbjEc MBoGA1UEAxMTQ0lMb2dvbiBPcGVuSUQgQ0EgMTAeFw0xMjEwMTcwMDE2NDlaFw0xMjEwMTcxMjIx NDlaMGkxEzARBgoJkiaJk/IsZAEZEwNvcmcxFzAVBgoJkiaJk/IsZAEZEwdjaWxvZ29uMQswCQYD VQQGEwJVUzEPMA0GA1UEChMGR29vZ2xlMRswGQYDVQQDExJPd2VuIE93bmVycmVwIEE4OTMwggEi MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYLdpgg88sntivH+af4oamlp7blsUQcCQ5Yc/b VDP/dwEKfxTcW36tMV3asLO7GcL7z4FESG761LAe86siT9rcwg2ttLkRjI9KeA3sFjC28N8XjKZ1 estCqG3odqw2pjo3VEFaU57219vIYMJhjmHKEgSnlMQeChMYun/sYIO5uNFba9BfiB6/PRS+bgee cXRsIAm1vkB89AHdEjqdvH0uSN+jGjF6aAPXsESh70DUAHzs14lbFAomig7AZafT+weh0G5pnayC lutVnhb9SyS3s1+A6kx8z9mkDUwY/NKXisuDeXa+WbRVq51D+Lc7ffOI+Ph+ynyfFGMcCBzbMADX AgMBAAGjgeEwgd4wDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCBLAwEwYDVR0lBAwwCgYIKwYB BQUHAwIwGAYDVR0gBBEwDzANBgsrBgEEAYKRNgEDAzBsBgNVHR8EZTBjMC+gLaArhilodHRwOi8v Y3JsLmNpbG9nb24ub3JnL2NpbG9nb24tb3BlbmlkLmNybDAwoC6gLIYqaHR0cDovL2NybC5kb2Vn cmlkcy5vcmcvY2lsb2dvbi1vcGVuaWQuY3JsMCEGA1UdEQQaMBiBFm93ZW5vd25lcnJlcEBnbWFp bC5jb20wDQYJKoZIhvcNAQELBQADggEBAHWd6ZOjSmJyOUyyLgZAPJpkSuk7DT5mFRhszJhfTGnu gANHRIJZMs5e/LCMypE+ftxb8mnhAE+kURA2DmeucazHUDP5oYofU+8KMYqcNKnPpLnuiw+bCJPa 3BDxrYoi+vVislHb0U+QDjVYtUtQ2b1/Xhv8ShH89O9i65bbOq+sqez6z2AD9RWOEwRwpQLc9D65 9lkrsKGmJtuG8q3NTpZ1DSuaLOtn0QqttdmCg3pu5edRtgdpGadaSGR4s222JasV439bSTL8Z0Ug HtjSclGqi8IBmvRkTZI61zTVbGdOKMP90LV1p8noJVLRkZpWRjLxI5xy9El8daAWMdjfrSc= -----END CERTIFICATE-----""" subject_2 = "/DC=org/DC=cilogon/C=US/O=Google/CN=Owen Ownerrep A893" # Try to merge with nonexistent email account with self.assertRaises(NotFound): self.identity_management_service.initiate_account_merge("*****@*****.**") with self.assertRaises(BadRequest): self.identity_management_service.initiate_account_merge() # Create two users id, valid_until, registered = self.identity_management_service.signon(certificate, True) self.assertFalse(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon(certificate_2, True) self.assertFalse(registered_2) # Validate the two accounts are different self.assertNotEqual(id, id_2, "The two accounts should have two different user id") # Create UserInfo contact_info_obj = IonObject("ContactInformation",{"email": "*****@*****.**"}) user_info_obj = IonObject("UserInfo", {"name": "Dude", "contact": contact_info_obj}) user_info_id = self.identity_management_service.create_user_info(id, user_info_obj) contact_info_obj_2 = IonObject("ContactInformation",{"email": "*****@*****.**"}) user_info_obj_2 = IonObject("UserInfo", {"name": "theDude", "contact": contact_info_obj_2}) user_info_id_2 = self.identity_management_service.create_user_info(id_2, user_info_obj_2) # Make sure the two users are registered id, valid_until, registered = self.identity_management_service.signon(certificate, True) self.assertTrue(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon(certificate_2, True) self.assertTrue(registered_2) token = self.identity_management_service.initiate_account_merge("*****@*****.**", headers={'ion-actor-id':id}) # Try merging accounts with invalid token string with self.assertRaises(NotFound): self.identity_management_service.complete_account_merge(token_string="0xBeeF", headers={'ion-actor-id':id}) with self.assertRaises(BadRequest): self.identity_management_service.complete_account_merge() # Try merging accounts with a different user # Since this user hasn't initiated account merge, the token doesn't exist in his/her UserInfo with self.assertRaises(NotFound): self.identity_management_service.complete_account_merge(token, headers={'ion-actor-id':id_2}) self.identity_management_service.complete_account_merge(token, headers={'ion-actor-id':id}) # Try merging the account again with self.assertRaises(BadRequest): self.identity_management_service.complete_account_merge(token, headers={'ion-actor-id':id}) # Signon again and verify the two accounts have been merged id, valid_until, registered = self.identity_management_service.signon(certificate, True) self.assertTrue(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon(certificate_2, True) self.assertTrue(registered_2) # Validate the two accounts are the same self.assertEqual(id, id_2, "The two accounts should have the same id") # Try to merge to your own account with self.assertRaises(BadRequest): token = self.identity_management_service.initiate_account_merge("*****@*****.**", headers={'ion-actor-id':id}) # Done testing. Delete user self.identity_management_service.delete_user_info(user_info_id) self.identity_management_service.unregister_user_credentials(id, subject) self.identity_management_service.delete_actor_identity(id)
class ProcessDispatcherServiceIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2cei.yml') self.rr_cli = ResourceRegistryServiceClient() self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node) self.process_definition = ProcessDefinition(name='test_process') self.process_definition.executable = { 'module': 'ion.services.cei.test.test_process_dispatcher', 'class': 'TestProcess' } self.process_definition_id = self.pd_cli.create_process_definition( self.process_definition) self.waiter = ProcessStateWaiter() def tearDown(self): self.waiter.stop() def test_create_schedule_cancel(self): process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS proc_name = 'myreallygoodname' pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid, name=proc_name) self.assertEqual(pid, pid2) # verifies L4-CI-CEI-RQ141 and L4-CI-CEI-RQ142 self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, {}) self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING) # make sure process is readable directly from RR (mirrored) # verifies L4-CI-CEI-RQ63 # verifies L4-CI-CEI-RQ64 proc = self.rr_cli.read(pid) self.assertEqual(proc.process_id, pid) # now try communicating with the process to make sure it is really running test_client = TestClient() for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # verifies L4-CI-CEI-RQ147 # check the process name was set in container got_proc_name = test_client.get_process_name(pid=pid2) self.assertEqual(proc_name, got_proc_name) # kill the process and start it again self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # kill the process for good self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) def test_schedule_with_config(self): process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) # verifies L4-CI-CEI-RQ66 # feed in a string that the process will return -- verifies that # configuration actually makes it to the instantiated process test_response = uuid.uuid4().hex configuration = {"test_response": test_response} pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration=configuration, process_id=pid) self.assertEqual(pid, pid2) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) test_client = TestClient() # verifies L4-CI-CEI-RQ139 # assure that configuration block (which can contain inputs, outputs, # and arbitrary config) 1) makes it to the process and 2) is returned # in process queries self.assertEqual(test_client.query(), test_response) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, configuration) # kill the process for good self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) def test_schedule_bad_config(self): process_schedule = ProcessSchedule() # a non-JSON-serializable IonObject o = ProcessTarget() with self.assertRaises(BadRequest) as ar: self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={"bad": o}) self.assertTrue(ar.exception.message.startswith("bad configuration")) def test_cancel_notfound(self): with self.assertRaises(NotFound): self.pd_cli.cancel_process("not-a-real-process-id") def test_create_invalid_definition(self): # create process definition missing module and class # verifies L4-CI-CEI-RQ137 executable = dict(url="http://somewhere.com/something.py") definition = ProcessDefinition(name="test_process", executable=executable) with self.assertRaises(BadRequest): self.pd_cli.create_process_definition(definition)
class TestCTDPChain(IonIntegrationTestCase): def setUp(self): super(TestCTDPChain, self).setUp() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.pubsub = PubsubManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.data_process_management = DataProcessManagementServiceClient() self.dataproduct_management = DataProductManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() # This is for the time values inside the packets going into the transform self.i = 0 self.cnt = 0 # Cleanup of queue created by the subscriber self.queue_cleanup = [] self.data_process_cleanup = [] def _get_new_ctd_L0_packet(self, stream_definition_id, length): rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) rdt['time'] = numpy.arange(self.i, self.i+length) for field in rdt: if isinstance(rdt._pdict.get_context(field).param_type, QuantityType): rdt[field] = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)]) g = rdt.to_granule() self.i+=length return g def clean_queues(self): for queue in self.queue_cleanup: xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() def cleaning_operations(self): for dproc_id in self.data_process_cleanup: self.data_process_management.delete_data_process(dproc_id) def test_ctdp_chain(self): """ Test that packets are processed by a chain of CTDP transforms: L0, L1 and L2 """ #------------------------------------------------------------------------------------- # Prepare the stream def to be used for transform chain #------------------------------------------------------------------------------------- #todo Check whether the right parameter dictionary is being used self._prepare_stream_def_for_transform_chain() #------------------------------------------------------------------------------------- # Prepare the data proc defs and in and out data products for the transforms #------------------------------------------------------------------------------------- # list_args_L0 = [data_proc_def_id, input_dpod_id, output_dpod_id] list_args_L0 = self._prepare_things_you_need_to_launch_transform(name_of_transform='L0') list_args_L1 = self._prepare_things_you_need_to_launch_transform(name_of_transform='L1') list_args_L2_density = self._prepare_things_you_need_to_launch_transform(name_of_transform='L2_density') list_args_L2_salinity = self._prepare_things_you_need_to_launch_transform(name_of_transform='L2_salinity') log.debug("Got the following args: L0 = %s, L1 = %s, L2 density = %s, L2 salinity = %s", list_args_L0, list_args_L1, list_args_L2_density, list_args_L2_salinity ) #------------------------------------------------------------------------------------- # Launch the CTDP transforms #------------------------------------------------------------------------------------- L0_data_proc_id = self._launch_transform('L0', *list_args_L0) L1_data_proc_id = self._launch_transform('L1', *list_args_L1) L2_density_data_proc_id = self._launch_transform('L2_density', *list_args_L2_density) L2_salinity_data_proc_id = self._launch_transform('L2_salinity', *list_args_L2_salinity) log.debug("Launched the transforms: L0 = %s, L1 = %s", L0_data_proc_id, L1_data_proc_id) #------------------------------------------------------------------------- # Start a subscriber listening to the output of each of the transforms #------------------------------------------------------------------------- ar_L0 = self.start_subscriber_listening_to_L0_transform(out_data_prod_id=list_args_L0[2]) ar_L1 = self.start_subscriber_listening_to_L1_transform(out_data_prod_id=list_args_L1[2]) ar_L2_density = self.start_subscriber_listening_to_L2_density_transform(out_data_prod_id=list_args_L2_density[2]) ar_L2_salinity = self.start_subscriber_listening_to_L2_density_transform(out_data_prod_id=list_args_L2_salinity[2]) #------------------------------------------------------------------- # Publish the parsed packets that the L0 transform is listening for #------------------------------------------------------------------- stream_id, stream_route = self.get_stream_and_route_for_data_prod(data_prod_id= list_args_L0[1]) self._publish_for_L0_transform(stream_id, stream_route) #------------------------------------------------------------------- # Check the granules being outputted by the transforms #------------------------------------------------------------------- self._check_granule_from_L0_transform(ar_L0) self._check_granule_from_L1_transform(ar_L1) self._check_granule_from_L2_density_transform(ar_L2_density) self._check_granule_from_L2_salinity_transform(ar_L2_salinity) def _prepare_stream_def_for_transform_chain(self): # Get the stream definition for the stream using the parameter dictionary # pdict_id = self.dataset_management.read_parameter_dictionary_by_name(parameter_dict_name, id_only=True) pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'input_param_for_L0') self.in_stream_def_id_for_L0 = self.pubsub.create_stream_definition(name='stream_def_for_L0', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub.delete_stream_definition, self.in_stream_def_id_for_L0) pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'params_for_other_transforms') self.stream_def_id = self.pubsub.create_stream_definition(name='stream_def_for_CTDBP_transforms', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub.delete_stream_definition, self.stream_def_id) log.debug("Got the parsed parameter dictionary: id: %s", pdict_id) log.debug("Got the stream def for parsed input to L0: %s", self.in_stream_def_id_for_L0) log.debug("Got the stream def for other other streams: %s", self.stream_def_id) def _prepare_things_you_need_to_launch_transform(self, name_of_transform = ''): module, class_name = self._get_class_module(name_of_transform) #------------------------------------------------------------------------- # Data Process Definition #------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name= 'CTDBP_%s_Transform' % name_of_transform, description= 'Data Process Definition for the CTDBP %s transform.' % name_of_transform, module= module, class_name=class_name) data_proc_def_id = self.data_process_management.create_data_process_definition(dpd_obj) self.addCleanup(self.data_process_management.delete_data_process_definition, data_proc_def_id) log.debug("created data process definition: id = %s", data_proc_def_id) #------------------------------------------------------------------------- # Construct temporal and spatial Coordinate Reference System objects for the data product objects #------------------------------------------------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() #------------------------------------------------------------------------- # Get the names of the input and output data products #------------------------------------------------------------------------- input_dpod_id = '' output_dpod_id = '' if name_of_transform == 'L0': input_dpod_id = self._create_input_data_product('parsed', tdom, sdom) output_dpod_id = self._create_output_data_product('L0', tdom, sdom) self.in_prod_for_L1 = output_dpod_id elif name_of_transform == 'L1': input_dpod_id = self.in_prod_for_L1 output_dpod_id = self._create_output_data_product('L1', tdom, sdom) self.in_prod_for_L2 = output_dpod_id elif name_of_transform == 'L2_density': input_dpod_id = self.in_prod_for_L2 output_dpod_id = self._create_output_data_product('L2_density', tdom, sdom) elif name_of_transform == 'L2_salinity': input_dpod_id = self.in_prod_for_L2 output_dpod_id = self._create_output_data_product('L2_salinity', tdom, sdom) else: self.fail("something bad happened") return [data_proc_def_id, input_dpod_id, output_dpod_id] def _get_class_module(self, name_of_transform): options = {'L0' : self._class_module_L0, 'L1' : self._class_module_L1, 'L2_density' : self._class_module_L2_density, 'L2_salinity' : self._class_module_L2_salinity} return options[name_of_transform]() def _class_module_L0(self): module = 'ion.processes.data.transforms.ctdbp.ctdbp_L0' class_name = 'CTDBP_L0_all' return module, class_name def _class_module_L1(self): module = 'ion.processes.data.transforms.ctdbp.ctdbp_L1' class_name = 'CTDBP_L1_Transform' return module, class_name def _class_module_L2_density(self): module = 'ion.processes.data.transforms.ctdbp.ctdbp_L2_density' class_name = 'CTDBP_DensityTransform' return module, class_name def _class_module_L2_salinity(self): module = 'ion.processes.data.transforms.ctdbp.ctdbp_L2_salinity' class_name = 'CTDBP_SalinityTransform' return module, class_name def _create_input_data_product(self, name_of_transform = '', tdom = None, sdom = None): dpod_obj = IonObject(RT.DataProduct, name='dprod_%s' % name_of_transform, description='for_%s' % name_of_transform, temporal_domain = tdom, spatial_domain = sdom) log.debug("the stream def id: %s", self.stream_def_id) if name_of_transform == 'L0': stream_def_id = self.in_stream_def_id_for_L0 else: stream_def_id = self.stream_def_id dpod_id = self.dataproduct_management.create_data_product(data_product=dpod_obj, stream_definition_id= stream_def_id ) self.addCleanup(self.dataproduct_management.delete_data_product, dpod_id) log.debug("got the data product out. id: %s", dpod_id) return dpod_id def _create_output_data_product(self, name_of_transform = '', tdom = None, sdom = None): dpod_obj = IonObject(RT.DataProduct, name='dprod_%s' % name_of_transform, description='for_%s' % name_of_transform, temporal_domain = tdom, spatial_domain = sdom) if name_of_transform == 'L0': stream_def_id = self.in_stream_def_id_for_L0 else: stream_def_id = self.stream_def_id dpod_id = self.dataproduct_management.create_data_product(data_product=dpod_obj, stream_definition_id=stream_def_id ) self.addCleanup(self.dataproduct_management.delete_data_product, dpod_id) return dpod_id def _launch_transform(self, name_of_transform = '', data_proc_def_id = None, input_dpod_id = None, output_dpod_id = None): # We need the key name here to be "L2_stream", since when the data process is launched, this name goes into # the config as in config.process.publish_streams.L2_stream when the config is used to launch the data process if name_of_transform in ['L0', 'L1']: binding = '%s_stream' % name_of_transform elif name_of_transform == 'L2_salinity': binding = 'salinity' elif name_of_transform == 'L2_density': binding = 'density' config = None if name_of_transform == 'L1': config = self._create_calibration_coefficients_dict() elif name_of_transform == 'L2_density': config = DotDict() config.process = {'lat' : 32.7153, 'lon' : 117.1564} log.debug("launching transform for name: %s",name_of_transform ) log.debug("launching transform for data_proc_def_id: %s\ninput_dpod_id: %s\noutput_dpod_id: %s", data_proc_def_id, input_dpod_id, output_dpod_id ) data_proc_id = self.data_process_management.create_data_process( data_process_definition_id = data_proc_def_id, in_data_product_ids= [input_dpod_id], out_data_product_ids = [output_dpod_id], configuration = config) self.addCleanup(self.data_process_management.delete_data_process, data_proc_id) self.data_process_management.activate_data_process(data_proc_id) self.addCleanup(self.data_process_management.deactivate_data_process, data_proc_id) log.debug("Created a data process for ctdbp %s transform: id = %s", name_of_transform, data_proc_id) return data_proc_id def get_stream_and_route_for_data_prod(self, data_prod_id = ''): stream_ids, _ = self.resource_registry.find_objects(data_prod_id, PRED.hasStream, RT.Stream, True) stream_id = stream_ids[0] input_stream = self.resource_registry.read(stream_id) stream_route = input_stream.stream_route return stream_id, stream_route def start_subscriber_listening_to_L0_transform(self, out_data_prod_id = ''): #----------- Create subscribers to listen to the two transforms -------------------------------- stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True) output_stream_id_of_transform = stream_ids[0] ar_L0 = self._start_subscriber_to_transform( name_of_transform = 'L0',stream_id=output_stream_id_of_transform) return ar_L0 def start_subscriber_listening_to_L1_transform(self, out_data_prod_id = ''): #----------- Create subscribers to listen to the two transforms -------------------------------- stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True) output_stream_id_of_transform = stream_ids[0] ar_L1 = self._start_subscriber_to_transform( name_of_transform = 'L1',stream_id=output_stream_id_of_transform) return ar_L1 def start_subscriber_listening_to_L2_density_transform(self, out_data_prod_id = ''): #----------- Create subscribers to listen to the two transforms -------------------------------- stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True) output_stream_id_of_transform = stream_ids[0] ar_L2_density = self._start_subscriber_to_transform( name_of_transform = 'L2_density', stream_id=output_stream_id_of_transform) return ar_L2_density def start_subscriber_listening_to_L2_salinity_transform(self, out_data_prod_id = ''): #----------- Create subscribers to listen to the two transforms -------------------------------- stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True) output_stream_id_of_transform = stream_ids[0] ar_L2_density = self._start_subscriber_to_transform( name_of_transform = 'L2_salinity',stream_id=output_stream_id_of_transform) return ar_L2_density def _start_subscriber_to_transform(self, name_of_transform = '', stream_id = ''): ar = gevent.event.AsyncResult() def subscriber(m,r,s): ar.set(m) sub = StandaloneStreamSubscriber(exchange_name='sub_%s' % name_of_transform, callback=subscriber) # Note that this running the below line creates an exchange since none of that name exists before sub_id = self.pubsub.create_subscription('subscriber_to_transform_%s' % name_of_transform, stream_ids=[stream_id], exchange_name='sub_%s' % name_of_transform) self.addCleanup(self.pubsub.delete_subscription, sub_id) self.pubsub.activate_subscription(sub_id) self.addCleanup(self.pubsub.deactivate_subscription, sub_id) sub.start() self.addCleanup(sub.stop) return ar def _check_granule_from_L0_transform(self, ar = None): granule_from_transform = ar.get(timeout=20) log.debug("Got the following granule from the L0 transform: %s", granule_from_transform) # Check the algorithm being applied self._check_application_of_L0_algorithm(granule_from_transform) def _check_granule_from_L1_transform(self, ar = None): granule_from_transform = ar.get(timeout=20) log.debug("Got the following granule from the L1 transform: %s", granule_from_transform) # Check the algorithm being applied self._check_application_of_L1_algorithm(granule_from_transform) def _check_granule_from_L2_density_transform(self, ar = None): granule_from_transform = ar.get(timeout=20) log.debug("Got the following granule from the L2 transform: %s", granule_from_transform) # Check the algorithm being applied self._check_application_of_L2_density_algorithm(granule_from_transform) def _check_granule_from_L2_salinity_transform(self, ar = None): granule_from_transform = ar.get(timeout=20) log.debug("Got the following granule from the L2 transform: %s", granule_from_transform) # Check the algorithm being applied self._check_application_of_L2_salinity_algorithm(granule_from_transform) def _check_application_of_L0_algorithm(self, granule = None): """ Check the algorithm applied by the L0 transform """ rdt = RecordDictionaryTool.load_from_granule(granule) list_of_expected_keys = ['time', 'pressure', 'conductivity', 'temperature'] for key in list_of_expected_keys: self.assertIn(key, rdt) def _check_application_of_L1_algorithm(self, granule = None): """ Check the algorithm applied by the L1 transform """ rdt = RecordDictionaryTool.load_from_granule(granule) list_of_expected_keys = [ 'time', 'pressure', 'conductivity', 'temp'] for key in list_of_expected_keys: self.assertIn(key, rdt) def _check_application_of_L2_density_algorithm(self, granule = None): """ Check the algorithm applied by the L2 transform """ rdt = RecordDictionaryTool.load_from_granule(granule) list_of_expected_keys = ['time', 'density'] for key in list_of_expected_keys: self.assertIn(key, rdt) def _check_application_of_L2_salinity_algorithm(self, granule = None): """ Check the algorithm applied by the L2 transform """ rdt = RecordDictionaryTool.load_from_granule(granule) list_of_expected_keys = ['time', 'salinity'] for key in list_of_expected_keys: self.assertIn(key, rdt) def _publish_for_L0_transform(self, input_stream_id = None, stream_route = None): #----------- Publish on that stream so that the transform can receive it -------------------------------- self._publish_to_transform(input_stream_id, stream_route ) def _publish_to_transform(self, stream_id = '', stream_route = None): pub = StandaloneStreamPublisher(stream_id, stream_route) publish_granule = self._get_new_ctd_L0_packet(stream_definition_id=self.in_stream_def_id_for_L0, length = 5) pub.publish(publish_granule) log.debug("Published the following granule: %s", publish_granule) def _create_input_param_dict_for_test(self, parameter_dict_name = ''): pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1900' pdict.add_context(t_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'Siemens_per_meter' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'Pascal' pdict.add_context(pres_ctxt) if parameter_dict_name == 'input_param_for_L0': temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) else: temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_kelvin' pdict.add_context(temp_ctxt) dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) dens_ctxt.uom = 'g/m' pdict.add_context(dens_ctxt) sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) sal_ctxt.uom = 'PSU' pdict.add_context(sal_ctxt) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in pdict.iteritems(): ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump()) pc_list.append(ctxt_id) if parameter_dict_name == 'input_param_for_L0': self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id) elif pc[1].name == 'temp': self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id) pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list) self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) return pdict_id def _create_calibration_coefficients_dict(self): config = DotDict() config.process.calibration_coeffs = { 'temp_calibration_coeffs': { 'TA0' : 1.561342e-03, 'TA1' : 2.561486e-04, 'TA2' : 1.896537e-07, 'TA3' : 1.301189e-07, 'TOFFSET' : 0.000000e+00 }, 'cond_calibration_coeffs': { 'G' : -9.896568e-01, 'H' : 1.316599e-01, 'I' : -2.213854e-04, 'J' : 3.292199e-05, 'CPCOR' : -9.570000e-08, 'CTCOR' : 3.250000e-06, 'CSLOPE' : 1.000000e+00 }, 'pres_calibration_coeffs' : { 'PA0' : 4.960417e-02, 'PA1' : 4.883682e-04, 'PA2' : -5.687309e-12, 'PTCA0' : 5.249802e+05, 'PTCA1' : 7.595719e+00, 'PTCA2' : -1.322776e-01, 'PTCB0' : 2.503125e+01, 'PTCB1' : 5.000000e-05, 'PTCB2' : 0.000000e+00, 'PTEMPA0' : -6.431504e+01, 'PTEMPA1' : 5.168177e+01, 'PTEMPA2' : -2.847757e-01, 'POFFSET' : 0.000000e+00 } } return config
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) #@unittest.skip('not working') def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', deviceAttachment #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- driver_config = { 'dvr_mod' : 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls' : 'SBE37Driver', 'workdir' : '/tmp/', } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) print 'TestDataProcessWithLookupTable: new Stream Definition id = ', instDevice_id print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test') try: ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) print 'TestDataProcessWithLookupTable: Data product streams1 = ', stream_ids #------------------------------- # Create CTD Raw as the second data product #------------------------------- print 'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition' raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def) dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test') try: ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new ctd_raw_data_product_id = ', ctd_raw_data_product self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) ) processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) ) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products={} log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity') ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True) log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure') ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True) log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature') ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, in_prods, self.output_products) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProcessWithLookupTable: create L0 all data_process return") contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) print 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = ', processAttachment
class TestAgentLaunchOps(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return def test_get_agent_client_noprocess(self): inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice)) iap = ResourceAgentClient._get_agent_process_id(inst_device_id) # should be no running agent self.assertIsNone(iap) # should raise NotFound self.assertRaises(NotFound, ResourceAgentClient, inst_device_id) def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = AgentProcessStateGate(self.PDC.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) if "BAD_DATA" == instance_obj.driver_config["comms_config"]: print "Saved config:" print snap_obj.content self.fail("Saved config was not properly restored") self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config_hasDevice(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(parent_device_id, PRED.hasDevice, child_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_objects(subject=parent_device_id, predicate=PRED.hasDevice, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasDevice") def test_agent_instance_config_hasNetworkParent(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(child_device_id, PRED.hasNetworkParent, parent_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_subjects(object=parent_device_id, predicate=PRED.hasNetworkParent, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasNetworkParent") def base_agent_instance_config(self, assign_child_platform_to_parent_platform_fn, find_child_platform_ids_of_parent_platform_fn): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC config_builder = DotDict config_builder.i = None config_builder.p = None def refresh_pconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.p = PlatformAgentConfigurationBuilder(clients) def refresh_iconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.i = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = [ {'lvl2': 'lvl3val'} ] required_config_keys = [ 'org_governance_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alerts_config', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) self.assertIn('ports', config['driver_config']) self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) self.assertIn('ports', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(name='', agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'}, 'alerts': generic_alerts_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.PlatformSite, name='sitePlatform') site_id = self.OMS.create_platform_site(platform_site=site_obj) # find current deployment using time constraints current_time = int( calendar.timegm(time.gmtime()) ) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-09-CTDMO0999', port_type=PortTypeEnum.UPLINK, ip_address=0) deployment_obj = IonObject(RT.Deployment, name='TestPlatformDeployment_' + name, description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds], port_assignments={platform_device_id:platform_port_obj}) deploy_id = self.OMS.create_deployment(deployment=deployment_obj, site_id=site_id, device_id=platform_device_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config, 'alerts': generic_alerts_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.InstrumentSite, name='siteInstrument') site_id = self.OMS.create_instrument_site(instrument_site =site_obj) # find current deployment using time constraints current_time = int( calendar.timegm(time.gmtime()) ) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-08-CTDMO0888', port_type=PortTypeEnum.PAYLOAD, ip_address=0) deployment_obj = IonObject(RT.Deployment, name='TestDeployment for Cabled Instrument', description='some new deployment', context=IonObject(OT.CabledInstrumentDeploymentContext), constraint_list=[temporal_bounds], port_assignments={instrument_device_id:platform_port_obj}) deploy_id = self.OMS.create_deployment(deployment=deployment_obj, site_id=site_id, device_id=instrument_device_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated self.assertRaises(AssertionError, config_builder.p.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure(name='child') platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_child_obj) child_config = config_builder.p.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure(name='parent') platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") assign_child_platform_to_parent_platform_fn(platform_device_child_id, platform_device_parent_id) child_device_ids = find_child_platform_ids_of_parent_platform_fn(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") refresh_iconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.i.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = config_builder.i.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_platform_device_using_has_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = config_builder.p.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) log.info("END base_agent_instance_config")
class TestIdentityManagementServiceInt(IonIntegrationTestCase): def setUp(self): self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient( node=self.container.node) self.identity_management_service = IdentityManagementServiceClient( node=self.container.node) self.org_client = OrgManagementServiceClient(node=self.container.node) def test_actor_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity( actor_identity_obj) actor_identity = self.identity_management_service.read_actor_identity( user_id) actor_identity.name = 'Updated subject' self.identity_management_service.update_actor_identity(actor_identity) ai = self.identity_management_service.find_actor_identity_by_name( actor_identity.name) self._baseAssertEqual(ai.name, actor_identity.name) with self.assertRaises(NotFound): ai = self.identity_management_service.find_actor_identity_by_name( "Yeah, well, you know, that's just, like, your opinion, man.") self._baseAssertEqual(ai.name, actor_identity.name) self.identity_management_service.delete_actor_identity(user_id) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_actor_identity(user_id) self.assertTrue("does not exist" in cm.exception.message) def test_user_credentials(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity( actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials( user_id, user_credentials_obj) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials( "bad", self.subject) self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials( user_id, "bad") self.assertTrue("does not exist" in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.unregister_user_credentials( 'bad', 'bad') self.assertTrue("does not exist" in cm.exception.message) self.identity_management_service.unregister_user_credentials( user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_user_info(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) user_id = self.identity_management_service.create_actor_identity( actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials( user_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info = self.identity_management_service.create_user_info( user_id, user_info_obj) with self.assertRaises(Conflict) as cm: self.identity_management_service.create_user_info( user_id, user_info_obj) self.assertTrue( "UserInfo already exists for user id" in cm.exception.message) user_info_obj = self.identity_management_service.find_user_info_by_id( user_id) user_info_obj = self.identity_management_service.find_user_info_by_name( "Foo") user_info_obj = self.identity_management_service.find_user_info_by_subject( self.subject) user_info_obj = self.identity_management_service.read_user_info( user_info) user_info_obj.name = 'Jane Doe' self.identity_management_service.update_user_info(user_info_obj) self.identity_management_service.delete_user_info(user_info) with self.assertRaises(NotFound) as cm: self.identity_management_service.read_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.delete_user_info(user_info) self.assertTrue('does not exist' in cm.exception.message) with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_name("John Doe") self.assertEqual(cm.exception.message, 'UserInfo with name John Doe does not exist') with self.assertRaises(NotFound) as cm: self.identity_management_service.find_user_info_by_subject( "Bogus subject") self.assertEqual( cm.exception.message, "UserCredentials with subject Bogus subject does not exist") self.identity_management_service.unregister_user_credentials( user_id, self.subject) self.identity_management_service.delete_actor_identity(user_id) def test_signon(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" id, valid_until, registered = self.identity_management_service.signon( certificate, True) self.assertFalse(registered) id2, valid_until2, registered2 = self.identity_management_service.signon( certificate, True) self.assertFalse(registered2) self.assertTrue(id == id2) self.assertTrue(valid_until == valid_until2) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) self.identity_management_service.create_user_info(id, user_info_obj) id3, valid_until3, registered3 = self.identity_management_service.signon( certificate, True) self.assertTrue(registered3) self.assertTrue(id == id3) self.assertTrue(valid_until == valid_until3) @attr('EXT') def test_get_extended_user_identity(self): actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) actor_id = self.identity_management_service.create_actor_identity( actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identity_management_service.register_user_credentials( actor_id, user_credentials_obj) user_info_obj = IonObject("UserInfo", {"name": "Foo"}) user_info_id = self.identity_management_service.create_user_info( actor_id, user_info_obj) ion_org = self.org_client.find_org() #Build the Service Agreement Proposal to to request a role but never close it sap = IonObject(OT.RequestRoleProposal, consumer=actor_id, provider=ion_org._id, role_name=ORG_MANAGER_ROLE) sap_response = self.org_client.negotiate(sap) #Just grant the role anyway #self.org_client.grant_role(ion_org._id, actor_id, ORG_MANAGER_ROLE) with self.assertRaises(NotFound): self.identity_management_service.get_user_info_extension( 'That rug really tied the room together.') with self.assertRaises(BadRequest): self.identity_management_service.get_user_info_extension() #Check the user without the negotiation role request extended_user = self.identity_management_service.get_user_info_extension( user_info_id, org_id=ion_org._id) self.assertEqual(user_info_obj.type_, extended_user.resource.type_) self.assertEqual(len(extended_user.roles), 1) self.assertEqual(len(extended_user.open_requests), 1) self.assertEqual(extended_user.open_requests[0].org_id, ion_org._id) self.assertEqual(extended_user.open_requests[0].user_id, user_info_id) self.assertEqual(extended_user.open_requests[0].request_type, OT.RequestRoleProposal) self.assertEqual(len(extended_user.closed_requests), 0) self.assertEqual(extended_user.open_requests[0]._id, extended_user.open_requests[0].negotiation_id) neg = self.resource_registry.read( object_id=extended_user.open_requests[0].negotiation_id) sap_response = Negotiation.create_counter_proposal( neg, ProposalStatusEnum.ACCEPTED, ProposalOriginatorEnum.PROVIDER) sap_response2 = self.org_client.negotiate(sap_response) #Now check the user after the negotiation has been accepted and the role granted extended_user = self.identity_management_service.get_user_info_extension( user_info_id, org_id=ion_org._id) self.assertEqual(user_info_obj.type_, extended_user.resource.type_) self.assertEqual(len(extended_user.roles), 2) self.assertEqual(len(extended_user.open_requests), 0) self.assertEqual(len(extended_user.closed_requests), 1) self.assertEqual(extended_user.closed_requests[0].org_id, ion_org._id) self.assertEqual(extended_user.closed_requests[0].user_id, user_info_id) self.assertEqual(extended_user.closed_requests[0].request_type, OT.RequestRoleProposal) self.identity_management_service.delete_user_info(user_info_id) self.org_client.revoke_role(org_id=ion_org._id, actor_id=actor_id, role_name=ORG_MANAGER_ROLE) self.identity_management_service.unregister_user_credentials( actor_id, self.subject) self.identity_management_service.delete_actor_identity(actor_id) def test_account_merge(self): certificate = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgICBQAwDQYJKoZIhvcNAQEFBQAwajETMBEGCgmSJomT8ixkARkWA29yZzEX MBUGCgmSJomT8ixkARkWB2NpbG9nb24xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdDSUxvZ29uMRsw GQYDVQQDExJDSUxvZ29uIEJhc2ljIENBIDEwHhcNMTAxMTE4MjIyNTA2WhcNMTAxMTE5MTAzMDA2 WjBvMRMwEQYKCZImiZPyLGQBGRMDb3JnMRcwFQYKCZImiZPyLGQBGRMHY2lsb2dvbjELMAkGA1UE BhMCVVMxFzAVBgNVBAoTDlByb3RlY3ROZXR3b3JrMRkwFwYDVQQDExBSb2dlciBVbndpbiBBMjU0 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6QhsWxhUXbIxg+1ZyEc7d+hIGvchVmtb g0kKLmivgoVsA4U7swNDRH6svW242THta0oTf6crkRx7kOKg6jma2lcAC1sjOSddqX7/92ChoUPq 7LWt2T6GVVA10ex5WAeB/o7br/Z4U8/75uCBis+ru7xEDl09PToK20mrkcz9M4HqIv1eSoPkrs3b 2lUtQc6cjuHRDU4NknXaVMXTBHKPM40UxEDHJueFyCiZJFg3lvQuSsAl4JL5Z8pC02T8/bODBuf4 dszsqn2SC8YDw1xrujvW2Bd7Q7BwMQ/gO+dZKM1mLJFpfEsR9WrjMeg6vkD2TMWLMr0/WIkGC8u+ 6M6SMQIDAQABo4HdMIHaMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgSwMBMGA1UdJQQMMAoG CCsGAQUFBwMCMBgGA1UdIAQRMA8wDQYLKwYBBAGCkTYBAgEwagYDVR0fBGMwYTAuoCygKoYoaHR0 cDovL2NybC5jaWxvZ29uLm9yZy9jaWxvZ29uLWJhc2ljLmNybDAvoC2gK4YpaHR0cDovL2NybC5k b2Vncmlkcy5vcmcvY2lsb2dvbi1iYXNpYy5jcmwwHwYDVR0RBBgwFoEUaXRzYWdyZWVuMUB5YWhv by5jb20wDQYJKoZIhvcNAQEFBQADggEBAEYHQPMY9Grs19MHxUzMwXp1GzCKhGpgyVKJKW86PJlr HGruoWvx+DLNX75Oj5FC4t8bOUQVQusZGeGSEGegzzfIeOI/jWP1UtIjzvTFDq3tQMNvsgROSCx5 CkpK4nS0kbwLux+zI7BWON97UpMIzEeE05pd7SmNAETuWRsHMP+x6i7hoUp/uad4DwbzNUGIotdK f8b270icOVgkOKRdLP/Q4r/x8skKSCRz1ZsRdR+7+B/EgksAJj7Ut3yiWoUekEMxCaTdAHPTMD/g Mh9xL90hfMJyoGemjJswG5g3fAdTP/Lv0I6/nWeH/cLjwwpQgIEjEAVXl7KHuzX5vPD/wqQ= -----END CERTIFICATE-----""" subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" certificate_2 = """-----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgIDAJ/lMA0GCSqGSIb3DQEBCwUAMGsxEzARBgoJkiaJk/IsZAEZFgNvcmcx FzAVBgoJkiaJk/IsZAEZFgdjaWxvZ29uMQswCQYDVQQGEwJVUzEQMA4GA1UEChMHQ0lMb2dvbjEc MBoGA1UEAxMTQ0lMb2dvbiBPcGVuSUQgQ0EgMTAeFw0xMjEwMTcwMDE2NDlaFw0xMjEwMTcxMjIx NDlaMGkxEzARBgoJkiaJk/IsZAEZEwNvcmcxFzAVBgoJkiaJk/IsZAEZEwdjaWxvZ29uMQswCQYD VQQGEwJVUzEPMA0GA1UEChMGR29vZ2xlMRswGQYDVQQDExJPd2VuIE93bmVycmVwIEE4OTMwggEi MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYLdpgg88sntivH+af4oamlp7blsUQcCQ5Yc/b VDP/dwEKfxTcW36tMV3asLO7GcL7z4FESG761LAe86siT9rcwg2ttLkRjI9KeA3sFjC28N8XjKZ1 estCqG3odqw2pjo3VEFaU57219vIYMJhjmHKEgSnlMQeChMYun/sYIO5uNFba9BfiB6/PRS+bgee cXRsIAm1vkB89AHdEjqdvH0uSN+jGjF6aAPXsESh70DUAHzs14lbFAomig7AZafT+weh0G5pnayC lutVnhb9SyS3s1+A6kx8z9mkDUwY/NKXisuDeXa+WbRVq51D+Lc7ffOI+Ph+ynyfFGMcCBzbMADX AgMBAAGjgeEwgd4wDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCBLAwEwYDVR0lBAwwCgYIKwYB BQUHAwIwGAYDVR0gBBEwDzANBgsrBgEEAYKRNgEDAzBsBgNVHR8EZTBjMC+gLaArhilodHRwOi8v Y3JsLmNpbG9nb24ub3JnL2NpbG9nb24tb3BlbmlkLmNybDAwoC6gLIYqaHR0cDovL2NybC5kb2Vn cmlkcy5vcmcvY2lsb2dvbi1vcGVuaWQuY3JsMCEGA1UdEQQaMBiBFm93ZW5vd25lcnJlcEBnbWFp bC5jb20wDQYJKoZIhvcNAQELBQADggEBAHWd6ZOjSmJyOUyyLgZAPJpkSuk7DT5mFRhszJhfTGnu gANHRIJZMs5e/LCMypE+ftxb8mnhAE+kURA2DmeucazHUDP5oYofU+8KMYqcNKnPpLnuiw+bCJPa 3BDxrYoi+vVislHb0U+QDjVYtUtQ2b1/Xhv8ShH89O9i65bbOq+sqez6z2AD9RWOEwRwpQLc9D65 9lkrsKGmJtuG8q3NTpZ1DSuaLOtn0QqttdmCg3pu5edRtgdpGadaSGR4s222JasV439bSTL8Z0Ug HtjSclGqi8IBmvRkTZI61zTVbGdOKMP90LV1p8noJVLRkZpWRjLxI5xy9El8daAWMdjfrSc= -----END CERTIFICATE-----""" subject_2 = "/DC=org/DC=cilogon/C=US/O=Google/CN=Owen Ownerrep A893" # Try to merge with nonexistent email account with self.assertRaises(NotFound): self.identity_management_service.initiate_account_merge( "*****@*****.**") with self.assertRaises(BadRequest): self.identity_management_service.initiate_account_merge() # Create two users id, valid_until, registered = self.identity_management_service.signon( certificate, True) self.assertFalse(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon( certificate_2, True) self.assertFalse(registered_2) # Validate the two accounts are different self.assertNotEqual( id, id_2, "The two accounts should have two different user id") # Create UserInfo contact_info_obj = IonObject("ContactInformation", {"email": "*****@*****.**"}) user_info_obj = IonObject("UserInfo", { "name": "Dude", "contact": contact_info_obj }) user_info_id = self.identity_management_service.create_user_info( id, user_info_obj) contact_info_obj_2 = IonObject("ContactInformation", {"email": "*****@*****.**"}) user_info_obj_2 = IonObject("UserInfo", { "name": "theDude", "contact": contact_info_obj_2 }) user_info_id_2 = self.identity_management_service.create_user_info( id_2, user_info_obj_2) # Make sure the two users are registered id, valid_until, registered = self.identity_management_service.signon( certificate, True) self.assertTrue(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon( certificate_2, True) self.assertTrue(registered_2) token = self.identity_management_service.initiate_account_merge( "*****@*****.**", headers={'ion-actor-id': id}) # Try merging accounts with invalid token string with self.assertRaises(NotFound): self.identity_management_service.complete_account_merge( token_string="0xBeeF", headers={'ion-actor-id': id}) with self.assertRaises(BadRequest): self.identity_management_service.complete_account_merge() # Try merging accounts with a different user # Since this user hasn't initiated account merge, the token doesn't exist in his/her UserInfo with self.assertRaises(NotFound): self.identity_management_service.complete_account_merge( token, headers={'ion-actor-id': id_2}) self.identity_management_service.complete_account_merge( token, headers={'ion-actor-id': id}) # Try merging the account again with self.assertRaises(BadRequest): self.identity_management_service.complete_account_merge( token, headers={'ion-actor-id': id}) # Signon again and verify the two accounts have been merged id, valid_until, registered = self.identity_management_service.signon( certificate, True) self.assertTrue(registered) id_2, valid_until_2, registered_2 = self.identity_management_service.signon( certificate_2, True) self.assertTrue(registered_2) # Validate the two accounts are the same self.assertEqual(id, id_2, "The two accounts should have the same id") # Try to merge to your own account with self.assertRaises(BadRequest): token = self.identity_management_service.initiate_account_merge( "*****@*****.**", headers={'ion-actor-id': id}) # Done testing. Delete user self.identity_management_service.delete_user_info(user_info_id) self.identity_management_service.unregister_user_credentials( id, subject) self.identity_management_service.delete_actor_identity(id)
class TestCTDTransformsIntegration(IonIntegrationTestCase): pdict_id = None def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataset_management = self.datasetclient def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration) return pid def _create_instrument_model(self): instModel_obj = IonObject( RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_agent_instance(self, instAgent_id,instDevice_id): port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) return instAgentInstance_id def _create_param_dicts(self): tdom, sdom = time_series_domain() self.sdom = sdom.dump() self.tdom = tdom.dump() self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) def _create_input_data_products(self, ctd_stream_def_id, instDevice_id, ): dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = self.tdom, spatial_domain = self.sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) #--------------------------------------------------------------------------- # Create CTD Raw as the second data product #--------------------------------------------------------------------------- if not self.pdict_id: self._create_param_dicts() raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=self.pdict_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = self.tdom, spatial_domain = self.sdom) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids return ctd_parsed_data_product def _create_data_process_definitions(self): #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L1 Conductivity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L1 Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L1 Temperature: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L2 Salinity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L2 Density: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) return self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id,\ self.ctd_L1_pressure_dprocdef_id,self.ctd_L1_temperature_dprocdef_id, \ self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id def _create_stream_definitions(self): if not self.pdict_id: self._create_param_dicts() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding= 'pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding='temperature' ) return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id def _create_l0_output_data_products(self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id): out_data_prods = [] ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) out_data_prods.append(self.ctd_l0_conductivity_output_dp_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_conductivity_output_dp_id) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) out_data_prods.append(self.ctd_l0_pressure_output_dp_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_pressure_output_dp_id) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) out_data_prods.append(self.ctd_l0_temperature_output_dp_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_temperature_output_dp_id) return out_data_prods def _create_l1_out_data_products(self): ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] ) self.loggerpids.append(pid) ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0] ) self.loggerpids.append(pid) ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0] ) self.loggerpids.append(pid) def _create_l2_out_data_products(self): #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- if not self.pdict_id: self._create_param_dicts() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding='density' ) ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0] ) self.loggerpids.append(pid) ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0] ) self.loggerpids.append(pid) @unittest.skip('This test errors on coi-nightly, may be OBE.') def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id,instDevice_id ) #------------------------------------------------------------------------------------- # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------- self._create_param_dicts() ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=self.pdict_id) #------------------------------------------------------------------------------------- # Create two data products #------------------------------------------------------------------------------------- ctd_parsed_data_product = self._create_input_data_products(ctd_stream_def_id,instDevice_id) #------------------------------------------------------------------------------------- # Create data process definitions #------------------------------------------------------------------------------------- self._create_data_process_definitions() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------------------------------------------------------------- outgoing_stream_l0_conductivity_id, \ outgoing_stream_l0_pressure_id, \ outgoing_stream_l0_temperature_id = self._create_stream_definitions() self.out_prod_ids = self._create_l0_output_data_products(outgoing_stream_l0_conductivity_id,outgoing_stream_l0_pressure_id,outgoing_stream_l0_temperature_id) self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding='pressure' ) self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding= 'temperature' ) self._create_l1_out_data_products() self._create_l2_out_data_products() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------------------------------------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = self.ctd_L0_all_dprocdef_id, in_data_product_ids = [ctd_parsed_data_product], out_data_product_ids = self.out_prod_ids) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process,process_ids[0]) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEquals(extended_process.computed.operational_state.status, ComputedValueAvailability.NOTAVAILABLE) self.assertEquals(data_process.message_controllable, True) #------------------------------------------------------------------------------------- # L1 Conductivity: Create the data process #------------------------------------------------------------------------------------- l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_conductivity_dprocdef_id, [self.ctd_l0_conductivity_output_dp_id], [self.ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) data_process = self.rrclient.read(l1_conductivity_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l1_conductivity_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process,process_ids[0]) #------------------------------------------------------------------------------------- # L1 Pressure: Create the data process #------------------------------------------------------------------------------------- l1_pressure_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_pressure_dprocdef_id, [self.ctd_l0_pressure_output_dp_id], [self.ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) data_process = self.rrclient.read(l1_pressure_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l1_pressure_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L1 Temperature: Create the data process #------------------------------------------------------------------------------------- l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_temperature_dprocdef_id, [self.ctd_l0_temperature_output_dp_id], [self.ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) data_process = self.rrclient.read(l1_temperature_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l1_temperature_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Salinity: Create the data process #------------------------------------------------------------------------------------- l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) data_process = self.rrclient.read(l2_salinity_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l2_salinity_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Density: Create the data process #------------------------------------------------------------------------------------- l2_density_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_density_output_dp_id]) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) data_process = self.rrclient.read(l2_density_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l2_density_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) # Wait for instrument agent to spawn gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Streaming #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #todo ResourceAgentClient no longer has method set_param # # Make sure the sampling rate and transmission are sane. # params = { # SBE37Parameter.NAVG : 1, # SBE37Parameter.INTERVAL : 5, # SBE37Parameter.TXREALTIME : True # } # self._ia_client.set_param(params) #todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it? cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) #todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it? # log.debug("test_activateInstrumentStream: calling go_observatory") # cmd = AgentCommand(command='go_observatory') # reply = self._ia_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #------------------------------------------------------------------------------------------------- # Cleanup processes #------------------------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) #-------------------------------------------------------------------------------- # Cleanup data products #-------------------------------------------------------------------------------- dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct, id_only=True) for dp_id in dp_ids: self.dataproductclient.delete_data_product(dp_id)
class TestDataProductVersions(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.client = DataProductManagementServiceClient( node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #@unittest.skip('not working') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}}) def test_createDataProductVersionSimple(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='test', parameter_dictionary_id=pdict_id) # test creating a new data product which will also create the initial/default version log.debug('Creating new data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP', description='some new dp', temporal_domain=tdom, spatial_domain=sdom) dp_id = self.client.create_data_product(dp_obj, ctd_stream_def_id) log.debug('new dp_id = %s', str(dp_id)) dpc_id = self.client.create_data_product_collection( data_product_id=dp_id, collection_name='firstCollection', collection_description='collection desc') #test that the links exist version_ids, _ = self.rrclient.find_objects(subject=dpc_id, predicate=PRED.hasVersion, id_only=True) log.debug('version_ids = %s', str(version_ids)) self.assertTrue( version_ids, 'Failed to connect the data product to the version collection.') self.assertTrue( version_ids[0] == dp_id, 'Failed to connect the data product to the version collection.') # test creating a subsequent data product version which will update the data product pointers dp2_obj = IonObject(RT.DataProduct, name='DP2', description='a second dp', temporal_domain=tdom, spatial_domain=sdom) dp2_id = self.client.create_data_product(dp2_obj, ctd_stream_def_id) log.debug('second dp_id = %s', dp2_id) log.debug("adding product %s to collection %s", dp2_id, dpc_id) self.client.add_data_product_version_to_collection( data_product_id=dp2_id, data_product_collection_id=dpc_id, version_name="second version", version_description="a second version created") #test that the links exist version_ids = self.RR2.find_data_product_ids_of_data_product_collection_using_has_version( dpc_id) self.assertEqual(2, len(version_ids)) recent_version_id = self.client.get_current_version(dpc_id) self.assertEquals(recent_version_id, dp2_id) base_version_id = self.client.get_base_version(dpc_id) self.assertEquals(base_version_id, dp_id) #--------------------------------------------------------------------------------------------- # Now check that we can subscribe to the stream for the data product version #--------------------------------------------------------------------------------------------- # Activating the data products contained in the versions held by the data product collection data_product_collection_obj = self.rrclient.read(dpc_id) version_list = data_product_collection_obj.version_list self.assertEquals(len(version_list), 2) for version in version_list: data_product_id = version.data_product_id self.client.activate_data_product_persistence(data_product_id) streams = self.RR2.find_streams_of_data_product_using_has_stream( data_product_id) self.assertNotEqual(0, len(streams)) for stream in streams: self.assertTrue(stream.persisted) log.debug( "This satisfies L4-CI-DM-RQ-053: 'The dynamic data distribution services shall support multiple " "versions of a given data topic.' This is true because we have shown above that we can persist the " "streams associated to the data product versions, and therefore we can subscribe to them. In " "test_oms_launch2.py, we have tested that activated data products like the ones we have here can be " "used by data processes to gather streaming data and it all works together correctly. This therefore " "completes the demonstration of req L4-CI-DM-RQ-053.") #--------------------------------------------------------------------------------------------- # Now delete all the created stuff and look for possible problems in doing so #--------------------------------------------------------------------------------------------- log.debug("deleting all data productgs") self.client.delete_data_product(dp_id) self.client.delete_data_product(dp2_id) self.client.delete_data_product_collection(dpc_id) self.client.force_delete_data_product_collection(dpc_id) # now try to get the deleted dp object self.assertRaises(NotFound, self.client.read_data_product_collection, dpc_id)
class TestExchangeManagementServiceInt(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2coi.yml') self.ems = ExchangeManagementServiceClient() self.rr = ResourceRegistryServiceClient() orglist, _ = self.rr.find_resources(RT.Org) if len(orglist) < 1: raise StandardError("No orgs found") self.org_id = orglist[0]._id # we test actual exchange interaction in pyon, so it's fine to mock the broker interaction here self.container.ex_manager.create_xs = Mock() self.container.ex_manager.delete_xs = Mock() self.container.ex_manager.create_xp = Mock() self.container.ex_manager.delete_xp = Mock() @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Test reaches into container, doesn\'t work with CEI') def test_xs_create_delete(self): exchange_space = ExchangeSpace(name="bobo") esid = self.ems.create_exchange_space(exchange_space, self.org_id) # should be able to pull from RR an exchange space es2 = self.rr.read(esid) self.assertEquals(exchange_space.name, es2.name) # should have an exchange declared on the broker self.container.ex_manager.create_xs.assert_called_once_with( 'bobo', use_ems=False) # should have an assoc to an org orglist, _ = self.rr.find_subjects(RT.Org, PRED.hasExchangeSpace, esid, id_only=True) self.assertEquals(len(orglist), 1) self.assertEquals(orglist[0], self.org_id) self.ems.delete_exchange_space(esid) # should no longer have that id in the RR self.assertRaises(NotFound, self.rr.read, esid) # should no longer have an assoc to an org orglist2, _ = self.rr.find_subjects(RT.Org, PRED.hasExchangeSpace, esid, id_only=True) self.assertEquals(len(orglist2), 0) # should no longer have that exchange declared self.assertEquals(self.container.ex_manager.delete_xs.call_count, 1) self.assertIn('bobo', self.container.ex_manager.delete_xs.call_args[0] [0].exchange) # prefixed with sysname def test_xs_delete_without_create(self): self.assertRaises(NotFound, self.ems.delete_exchange_space, '123') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Test reaches into container, doesn\'t work with CEI') def test_xp_create_delete(self): # xp needs an xs first exchange_space = ExchangeSpace(name="doink") esid = self.ems.create_exchange_space(exchange_space, self.org_id) exchange_point = ExchangePoint(name="hammer") epid = self.ems.create_exchange_point(exchange_point, esid) # should be in RR ep2 = self.rr.read(epid) self.assertEquals(exchange_point.name, ep2.name) # should be associated to the XS as well xslist, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangePoint, epid, id_only=True) self.assertEquals(len(xslist), 1) self.assertEquals(xslist[0], esid) # should exist on broker (both xp and xs) self.container.ex_manager.create_xs.assert_called_once_with( 'doink', use_ems=False) self.assertIn('hammer', self.container.ex_manager.create_xp.call_args[0]) self.ems.delete_exchange_point(epid) self.ems.delete_exchange_space(esid) # should no longer be in RR self.assertRaises(NotFound, self.rr.read, epid) # should no longer be associated xslist2, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangePoint, epid, id_only=True) self.assertEquals(len(xslist2), 0) # should no longer exist on broker (both xp and xs) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Test reaches into container, doesn\'t work with CEI') def test_xp_create_then_delete_xs(self): # xp needs an xs first exchange_space = ExchangeSpace(name="doink") esid = self.ems.create_exchange_space(exchange_space, self.org_id) exchange_point = ExchangePoint(name="hammer") epid = self.ems.create_exchange_point(exchange_point, esid) # delete xs self.ems.delete_exchange_space(esid) # should no longer have an association xslist2, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangePoint, epid, id_only=True) self.assertEquals(len(xslist2), 0) # TEST ONLY: have to clean up the xp or we leave junk on the broker # we have to do it manually because the xs is gone #self.ems.delete_exchange_point(epid) # @TODO: reaching into ex manager for transport is clunky xs = exchange.ExchangeSpace( self.container.ex_manager, self.container.ex_manager._priviledged_transport, exchange_space.name) xp = exchange.ExchangePoint( self.container.ex_manager, self.container.ex_manager._priviledged_transport, exchange_point.name, xs, 'ttree') self.container.ex_manager.delete_xp(xp, use_ems=False) def test_xn_declare_and_undeclare(self): # xn needs an xs first exchange_space = ExchangeSpace(name="bozo") esid = self.ems.create_exchange_space(exchange_space, self.org_id) exchange_name = ExchangeName(name='shoes', xn_type="XN_PROCESS") enid = self.ems.declare_exchange_name(exchange_name, esid) # should be in RR en2 = self.rr.read(enid) self.assertEquals(exchange_name.name, en2.name) # should have an assoc from XN to XS xnlist, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangeName, enid, id_only=True) self.assertEquals(len(xnlist), 1) self.assertEquals(xnlist[0], esid) # container API got called, will have declared a queue self.ems.undeclare_exchange_name( enid) # canonical name = xn id in current impl def test_xn_declare_no_xs(self): exchange_name = ExchangeName(name="shoez", xn_type='XN_PROCESS') self.assertRaises(NotFound, self.ems.declare_exchange_name, exchange_name, '11') def test_xn_undeclare_without_declare(self): self.assertRaises(NotFound, self.ems.undeclare_exchange_name, 'some_non_id') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Test reaches into container, doesn\'t work with CEI') def test_xn_declare_then_delete_xs(self): # xn needs an xs first exchange_space = ExchangeSpace(name="bozo") esid = self.ems.create_exchange_space(exchange_space, self.org_id) exchange_name = ExchangeName(name='shnoz', xn_type="XN_PROCESS") enid = self.ems.declare_exchange_name(exchange_name, esid) # delete the XS self.ems.delete_exchange_space(esid) # no longer should have assoc from XS to XN xnlist, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangeName, enid, id_only=True) self.assertEquals(len(xnlist), 0) # cleanup: delete the XN (assoc already removed, so we reach into the implementation here) # @TODO: reaching into ex manager for transport is clunky self.rr.delete(enid) xs = exchange.ExchangeSpace( self.container.ex_manager, self.container.ex_manager._priviledged_transport, exchange_space.name) xn = exchange.ExchangeName( self.container.ex_manager, self.container.ex_manager._priviledged_transport, exchange_name.name, xs) self.container.ex_manager.delete_xn(xn, use_ems=False)
class TransformManagementServiceIntTest(IonIntegrationTestCase): def setUp(self): # set up the container self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) self.tms_cli = TransformManagementServiceClient(node=self.container.node) self.rr_cli = ResourceRegistryServiceClient(node=self.container.node) self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node) self.input_stream_id = self.pubsub_cli.create_stream(name='input_stream',original=True) self.input_subscription_id = self.pubsub_cli.create_subscription(query=StreamQuery(stream_ids=[self.input_stream_id]),exchange_name='transform_input',name='input_subscription') self.output_stream_id = self.pubsub_cli.create_stream(name='output_stream',original=True) self.process_definition = ProcessDefinition(name='basic_transform_definition') self.process_definition.executable = {'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample'} self.process_definition_id = self.procd_cli.create_process_definition(process_definition=self.process_definition) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_create_transform(self): configuration = {'program_args':{'arg1':'value'}} transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id) # test transform creation in rr transform = self.rr_cli.read(transform_id) self.assertEquals(transform.name,'test_transform') # test associations predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition] assocs = [] for p in predicates: assocs += self.rr_cli.find_associations(transform_id,p,id_only=True) self.assertEquals(len(assocs),3) # test process creation transform = self.tms_cli.read_transform(transform_id) pid = transform.process_id proc = self.container.proc_manager.procs.get(pid) self.assertIsInstance(proc,TransformExample) # clean up self.tms_cli.delete_transform(transform_id) def test_create_transform_no_procdef(self): with self.assertRaises(NotFound): self.tms_cli.create_transform(name='test',in_subscription_id=self.input_subscription_id) def test_create_transform_bad_procdef(self): with self.assertRaises(NotFound): self.tms_cli.create_transform(name='test', in_subscription_id=self.input_subscription_id, process_definition_id='bad') def test_create_transform_no_config(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id, ) self.tms_cli.delete_transform(transform_id) def test_create_transform_name_failure(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id, ) with self.assertRaises(BadRequest): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id, ) self.tms_cli.delete_transform(transform_id) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_create_no_output(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, process_definition_id=self.process_definition_id, ) predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition] assocs = [] for p in predicates: assocs += self.rr_cli.find_associations(transform_id,p,id_only=True) self.assertEquals(len(assocs),2) # test process creation transform = self.tms_cli.read_transform(transform_id) pid = transform.process_id proc = self.container.proc_manager.procs.get(pid) self.assertIsInstance(proc,TransformExample) self.tms_cli.delete_transform(transform_id) def test_read_transform_exists(self): trans_obj = IonObject(RT.Transform,name='trans_obj') trans_id, _ = self.rr_cli.create(trans_obj) res = self.tms_cli.read_transform(trans_id) actual = self.rr_cli.read(trans_id) self.assertEquals(res._id,actual._id) def test_read_transform_nonexist(self): with self.assertRaises(NotFound) as e: self.tms_cli.read_transform('123') def test_activate_transform(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id ) self.tms_cli.activate_transform(transform_id) # pubsub check if activated? self.tms_cli.delete_transform(transform_id) def test_activate_transform_nonexist(self): with self.assertRaises(NotFound): self.tms_cli.activate_transform('1234') def test_delete_transform(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, process_definition_id=self.process_definition_id ) self.tms_cli.delete_transform(transform_id) # assertions with self.assertRaises(NotFound): self.rr_cli.read(transform_id) def test_delete_transform_nonexist(self): with self.assertRaises(NotFound): self.tms_cli.delete_transform('123') def test_execute_transform(self): # set up process_definition = ProcessDefinition(name='procdef_execute') process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example' process_definition.executable['class'] = 'ReverseTransform' data = [1,2,3] process_definition_id, _ = self.rr_cli.create(process_definition) retval = self.tms_cli.execute_transform(process_definition_id,data) self.assertEquals(retval,[3,2,1]) def test_integrated_transform(self): ''' This example script runs a chained three way transform: B A < C Where A is the even_odd transform (generates a stream of even and odd numbers from input) and B and C are the basic transforms that receive even and odd input ''' cc = self.container assertions = self.assertTrue pubsub_cli = PubsubManagementServiceClient(node=cc.node) rr_cli = ResourceRegistryServiceClient(node=cc.node) tms_cli = TransformManagementServiceClient(node=cc.node) #------------------------------- # Process Definition #------------------------------- # Create the process definition for the basic transform process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample' } basic_transform_definition_id, _ = rr_cli.create(process_definition) # Create The process definition for the TransformEvenOdd process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformEvenOdd' } evenodd_transform_definition_id, _ = rr_cli.create(process_definition) #------------------------------- # Streams #------------------------------- streams = [pubsub_cli.create_stream() for i in xrange(5)] #------------------------------- # Subscriptions #------------------------------- query = StreamQuery(stream_ids=[streams[0]]) input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue') query = StreamQuery(stream_ids = [streams[1]]) # even output even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue') query = StreamQuery(stream_ids = [streams[2]]) # odd output odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue') #------------------------------- # Launch the EvenOdd Transform #------------------------------- evenodd_id = tms_cli.create_transform(name='even_odd', in_subscription_id=input_subscription_id, out_streams={'even':streams[1], 'odd':streams[2]}, process_definition_id=evenodd_transform_definition_id, configuration={}) tms_cli.activate_transform(evenodd_id) #------------------------------- # Launch the Even Processing Transform #------------------------------- even_transform_id = tms_cli.create_transform(name='even_transform', in_subscription_id = even_subscription_id, out_streams={'even_plus1':streams[3]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(even_transform_id) #------------------------------- # Launch the Odd Processing Transform #------------------------------- odd_transform_id = tms_cli.create_transform(name='odd_transform', in_subscription_id = odd_subscription_id, out_streams={'odd_plus1':streams[4]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(odd_transform_id) #------------------------------- # Set up final subscribers #------------------------------- evenplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[3]]), exchange_name='evenplus1_queue', name='EvenPlus1Subscription', description='EvenPlus1 SubscriptionDescription' ) oddplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[4]]), exchange_name='oddplus1_queue', name='OddPlus1Subscription', description='OddPlus1 SubscriptionDescription' ) total_msg_count = 2 msgs = gevent.queue.Queue() def even1_message_received(message, headers): input = int(message.get('num')) assertions( (input % 2) ) # Assert it is odd (transform adds 1) msgs.put(True) def odd1_message_received(message, headers): input = int(message.get('num')) assertions(not (input % 2)) # Assert it is even msgs.put(True) subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node) even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received) odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received) # Start subscribers even_subscriber.start() odd_subscriber.start() # Activate subscriptions pubsub_cli.activate_subscription(evenplus1_subscription_id) pubsub_cli.activate_subscription(oddplus1_subscription_id) #------------------------------- # Set up fake stream producer #------------------------------- pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node) stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0]) #------------------------------- # Start test #------------------------------- # Publish a stream for i in xrange(total_msg_count): stream_publisher.publish({'num':str(i)}) time.sleep(0.5) for i in xrange(total_msg_count * 2): try: msgs.get() except Empty: assertions(False, "Failed to process all messages correctly.") """
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient self.processdispatchclient = ProcessDispatcherServiceClient(node = self.container.node) def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.info('test_createTransformsThenActivateInstrument: new InstrumentModel id = %s', instModel_id) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s', instDevice_id) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance" ) self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) log.info( 'TestDataProcessWithLookupTable: new Stream Definition id = %s', instDevice_id) log.info( 'Creating new CDM data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) log.info( 'new ctd_parsed_data_product_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.info('TestDataProcessWithLookupTable: Data product streams1 = %s', stream_ids) #------------------------------- # Create CTD Raw as the second data product #------------------------------- log.info('TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition') raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='ctd_raw', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) log.info( 'new ctd_raw_data_product_id = %s', ctd_raw_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.info( 'Data product streams2 = %s', stream_ids) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) ) processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) ) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) out_prods = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id,ctd_l0_temperature_output_dp_id] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L0_all_dprocdef_id, in_data_product_ids = in_prods, out_data_product_ids = out_prods) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProcessWithLookupTable: create L0 all data_process return") data_process= self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject= ctd_l0_all_data_process_id, predicate= PRED.hasProcess, object_type= RT.Process,id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s', processAttachment)
class CtdTransformsIntTest(IonIntegrationTestCase): def setUp(self): super(CtdTransformsIntTest, self).setUp() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.pubsub = PubsubManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.data_process_management = DataProcessManagementServiceClient() self.dataproduct_management = DataProductManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() # This is for the time values inside the packets going into the transform self.i = 0 # Cleanup of queue created by the subscriber self.queue_cleanup = [] self.data_process_cleanup = [] def _create_input_param_dict_for_test(self, parameter_dict_name = ''): pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1900' pdict.add_context(t_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = '' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = '' pdict.add_context(pres_ctxt) if parameter_dict_name == 'input_param_dict': temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) else: temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = '' pdict.add_context(temp_ctxt) dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) dens_ctxt.uom = '' pdict.add_context(dens_ctxt) sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) sal_ctxt.uom = '' pdict.add_context(sal_ctxt) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in pdict.iteritems(): ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump()) pc_list.append(ctxt_id) if parameter_dict_name == 'input_param_dict': self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id) elif parameter_dict_name == 'output_param_dict' and pc[1].name == 'temp': self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id) pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list) self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) return pdict_id def _get_new_ctd_L0_packet(self, stream_definition_id, length): rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) rdt['time'] = numpy.arange(self.i, self.i+length) for field in rdt: if isinstance(rdt._pdict.get_context(field).param_type, QuantityType): rdt[field] = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)]) g = rdt.to_granule() self.i+=length return g def _create_calibration_coefficients_dict(self): config = DotDict() config.process.calibration_coeffs = { 'temp_calibration_coeffs': { 'TA0' : 1.561342e-03, 'TA1' : 2.561486e-04, 'TA2' : 1.896537e-07, 'TA3' : 1.301189e-07, 'TOFFSET' : 0.000000e+00 }, 'cond_calibration_coeffs': { 'G' : -9.896568e-01, 'H' : 1.316599e-01, 'I' : -2.213854e-04, 'J' : 3.292199e-05, 'CPCOR' : -9.570000e-08, 'CTCOR' : 3.250000e-06, 'CSLOPE' : 1.000000e+00 }, 'pres_calibration_coeffs' : { 'PA0' : 4.960417e-02, 'PA1' : 4.883682e-04, 'PA2' : -5.687309e-12, 'PTCA0' : 5.249802e+05, 'PTCA1' : 7.595719e+00, 'PTCA2' : -1.322776e-01, 'PTCB0' : 2.503125e+01, 'PTCB1' : 5.000000e-05, 'PTCB2' : 0.000000e+00, 'PTEMPA0' : -6.431504e+01, 'PTEMPA1' : 5.168177e+01, 'PTEMPA2' : -2.847757e-01, 'POFFSET' : 0.000000e+00 } } return config def clean_queues(self): for queue in self.queue_cleanup: xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() def cleaning_operations(self): for dproc_id in self.data_process_cleanup: self.data_process_management.delete_data_process(dproc_id) def test_ctd_L1_all(self): """ Test that packets are processed by the ctd_L1_all transform """ #----------- Data Process Definition -------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='CTDBP_L1_Transform', description='Take granules on the L0 stream which have the C, T and P data and separately apply algorithms and output on the L1 stream.', module='ion.processes.data.transforms.ctdbp.ctdbp_L1', class_name='CTDBP_L1_Transform') dprocdef_id = self.data_process_management.create_data_process_definition(dpd_obj) self.addCleanup(self.data_process_management.delete_data_process_definition, dprocdef_id) log.debug("created data process definition: id = %s", dprocdef_id) #----------- Data Products -------------------------------- # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() # Get the stream definition for the stream using the parameter dictionary L0_pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'input_param_dict') L0_stream_def_id = self.pubsub.create_stream_definition(name='parsed', parameter_dictionary_id=L0_pdict_id) self.addCleanup(self.pubsub.delete_stream_definition, L0_stream_def_id) L1_pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'output_param_dict') L1_stream_def_id = self.pubsub.create_stream_definition(name='L1_out', parameter_dictionary_id=L1_pdict_id) self.addCleanup(self.pubsub.delete_stream_definition, L1_stream_def_id) log.debug("Got the parsed parameter dictionary: id: %s", L0_pdict_id) log.debug("Got the stream def for parsed input: %s", L0_stream_def_id) log.debug("got the stream def for the output: %s", L1_stream_def_id) # Input data product L0_stream_dp_obj = IonObject(RT.DataProduct, name='L0_stream', description='L0 stream input to CTBP L1 transform', temporal_domain = tdom, spatial_domain = sdom) input_dp_id = self.dataproduct_management.create_data_product(data_product=L0_stream_dp_obj, stream_definition_id=L0_stream_def_id ) self.addCleanup(self.dataproduct_management.delete_data_product, input_dp_id) # output data product L1_stream_dp_obj = IonObject(RT.DataProduct, name='L1_stream', description='L1_stream output of CTBP L1 transform', temporal_domain = tdom, spatial_domain = sdom) L1_stream_dp_id = self.dataproduct_management.create_data_product(data_product=L1_stream_dp_obj, stream_definition_id=L1_stream_def_id ) self.addCleanup(self.dataproduct_management.delete_data_product, L1_stream_dp_id) # We need the key name here to be "L1_stream", since when the data process is launched, this name goes into # the config as in config.process.publish_streams.L1_stream when the config is used to launch the data process self.output_products = {'L1_stream' : L1_stream_dp_id} out_stream_ids, _ = self.resource_registry.find_objects(L1_stream_dp_id, PRED.hasStream, RT.Stream, True) self.assertTrue(len(out_stream_ids)) output_stream_id = out_stream_ids[0] config = self._create_calibration_coefficients_dict() dproc_id = self.data_process_management.create_data_process( dprocdef_id, [input_dp_id], self.output_products, config) self.addCleanup(self.data_process_management.delete_data_process, dproc_id) log.debug("Created a data process for ctdbp_L1. id: %s", dproc_id) # Activate the data process self.data_process_management.activate_data_process(dproc_id) self.addCleanup(self.data_process_management.deactivate_data_process, dproc_id) #----------- Find the stream that is associated with the input data product when it was created by create_data_product() -------------------------------- stream_ids, _ = self.resource_registry.find_objects(input_dp_id, PRED.hasStream, RT.Stream, True) input_stream_id = stream_ids[0] input_stream = self.resource_registry.read(input_stream_id) stream_route = input_stream.stream_route log.debug("The input stream for the L1 transform: %s", input_stream_id) #----------- Create a subscriber that will listen to the transform's output -------------------------------- ar = gevent.event.AsyncResult() def subscriber(m,r,s): ar.set(m) sub = StandaloneStreamSubscriber(exchange_name='sub', callback=subscriber) sub_id = self.pubsub.create_subscription('subscriber_to_transform', stream_ids=[output_stream_id], exchange_name='sub') self.addCleanup(self.pubsub.delete_subscription, sub_id) self.pubsub.activate_subscription(sub_id) self.addCleanup(self.pubsub.deactivate_subscription, sub_id) sub.start() self.addCleanup(sub.stop) #----------- Publish on that stream so that the transform can receive it -------------------------------- pub = StandaloneStreamPublisher(input_stream_id, stream_route) publish_granule = self._get_new_ctd_L0_packet(stream_definition_id=L0_stream_def_id, length = 5) pub.publish(publish_granule) log.debug("Published the following granule: %s", publish_granule) granule_from_transform = ar.get(timeout=20) log.debug("Got the following granule from the transform: %s", granule_from_transform) # Check that the granule published by the L1 transform has the right properties self._check_granule_from_transform(granule_from_transform) def _check_granule_from_transform(self, granule): """ An internal method to check if a granule has the right properties """ rdt = RecordDictionaryTool.load_from_granule(granule) self.assertIn('pressure', rdt) self.assertIn('temp', rdt) self.assertIn('conductivity', rdt) self.assertIn('time', rdt)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @attr('EXT') @attr('PREP') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) # Assert that the data product has an associated stream def at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True) self.assertNotEquals(len(stream_ids), 0) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) group_names = self.dpsc_cli.get_data_product_group_list() self.assertIn("PRODNAME", group_names) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) def ion_object_encoder(obj): return obj.__dict__ #test prepare for create data_product_data = self.dpsc_cli.prepare_data_product_support() #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, "") self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0) #test prepare for update data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, dp_id) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1) self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1) self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) # Assert that there are no associated streams leftover after deleting the data product stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertEquals(len(stream_ids), 0) self.assertEquals(len(assoc_ids), 0) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_derived_data_product(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id) tdom, sdom = time_series_domain() dp = DataProduct(name='Instrument DP', temporal_domain=tdom.dump(), spatial_domain=sdom.dump()) dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id) self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id) self.dpsc_cli.activate_data_product_persistence(dp_id) self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Make the derived data product simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp']) tempwat_dp = DataProduct(name='TEMPWAT') tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id) self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) rdt['pressure'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True) tempwat_dataset_id = tempwat_dataset_ids[0] granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) self.assertEquals(set(rdt.fields), set(['time','temp'])) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug("The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'") data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name,'DP1') self.assertEquals(data_product_object.description,'some new dp') log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) dataset_modified.clear() rdt['time'] = np.arange(20,40) publisher.publish(rdt.to_granule()) self.assertFalse(dataset_modified.wait(2)) self.dpsc_cli.activate_data_product_persistence(dp_id) dataset_modified.clear() publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_almost_equal(rdt['time'], np.arange(40)) dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True) self.assertEquals(len(dataset_ids), 1) self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id) def test_lookup_values(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_lookups() stream_def_id = self.pubsubcli.create_stream_definition('lookup', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id) data_product = DataProduct(name='lookup data product') tdom, sdom = time_series_domain() data_product.temporal_domain = tdom.dump() data_product.spatial_domain = sdom.dump() data_product_id = self.dpsc_cli.create_data_product(data_product, stream_definition_id=stream_def_id) self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id) data_producer = DataProducer(name='producer') data_producer.producer_context = DataProcessProducerContext() data_producer.producer_context.configuration['qc_keys'] = ['offset_document'] data_producer_id, _ = self.rrclient.create(data_producer) self.addCleanup(self.rrclient.delete, data_producer_id) assoc,_ = self.rrclient.create_association(subject=data_product_id, object=data_producer_id, predicate=PRED.hasDataProducer) self.addCleanup(self.rrclient.delete_association, assoc) document_keys = self.damsclient.list_qc_references(data_product_id) self.assertEquals(document_keys, ['offset_document']) svm = StoredValueManager(self.container) svm.stored_value_cas('offset_document', {'offset_a':2.0}) self.dpsc_cli.activate_data_product_persistence(data_product_id) dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasDataset, id_only=True) dataset_id = dataset_ids[0] dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [0] rdt['temp'] = [20.] granule = rdt.to_granule() stream_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish(granule) self.assertTrue(dataset_monitor.event.wait(10)) granule = self.data_retriever.retrieve(dataset_id) rdt2 = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['temp'], rdt2['temp']) np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0])) svm.stored_value_cas('updated_document', {'offset_a':3.0}) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent) ep.publish_event(origin=data_product_id, reference_keys=['updated_document']) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [1] rdt['temp'] = [20.] granule = rdt.to_granule() gevent.sleep(2) # Yield so that the event goes through publisher.publish(granule) self.assertTrue(dataset_monitor.event.wait(10)) granule = self.data_retriever.retrieve(dataset_id) rdt2 = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt2['temp'],np.array([20.,20.])) np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0,23.0]))
class TestWorkflowManagementIntegration(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.workflowclient = WorkflowManagementServiceClient( node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient( node=self.container.node) self.ctd_stream_def = SBE37_CDM_stream_definition() def _create_ctd_input_stream_and_data_product( self, data_product_name='ctd_parsed'): cc = self.container assertions = self.assertTrue #------------------------------- # Create CTD Parsed as the initial data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition( container=self.ctd_stream_def, name='Simulated CTD data') log.debug('Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name=data_product_name, description='ctd stream test') try: ctd_parsed_data_product_id = self.dataproductclient.create_data_product( dp_obj, ctd_stream_def_id) except Exception as ex: self.fail("failed to create new data product: %s" % ex) log.debug('new ctd_parsed_data_product_id = ', ctd_parsed_data_product_id) #Only ever need one device for testing purposes. instDevice_obj, _ = self.rrclient.find_resources( restype=RT.InstrumentDevice, name='SBE37IMDevice') if instDevice_obj: instDevice_id = instDevice_obj[0]._id else: instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product_id) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_parsed_data_product_id, persist_data=False, persist_metadata=False) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) ctd_stream_id = stream_ids[0] return ctd_stream_id, ctd_parsed_data_product_id def _start_simple_input_stream_process(self, ctd_stream_id): return self._start_input_stream_process(ctd_stream_id) def _start_sinusoidal_input_stream_process(self, ctd_stream_id): return self._start_input_stream_process( ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher', 'SinusoidalCtdPublisher') def _start_input_stream_process( self, ctd_stream_id, module='ion.processes.data.ctd_stream_publisher', class_name='SimpleCtdPublisher'): ### ### Start the process for producing the CTD data ### # process definition for the ctd simulator... producer_definition = ProcessDefinition() producer_definition.executable = { 'module': module, 'class': class_name } ctd_sim_procdef_id = self.process_dispatcher.create_process_definition( process_definition=producer_definition) # Start the ctd simulator to produce some data configuration = { 'process': { 'stream_id': ctd_stream_id, } } ctd_sim_pid = self.process_dispatcher.schedule_process( process_definition_id=ctd_sim_procdef_id, configuration=configuration) return ctd_sim_pid def _start_output_stream_listener(self, data_product_stream_ids, message_count_per_stream=10): cc = self.container assertions = self.assertTrue ### ### Make a subscriber in the test to listen for transformed data ### salinity_subscription_id = self.pubsubclient.create_subscription( query=StreamQuery(data_product_stream_ids), exchange_name='workflow_test', name="test workflow transformations", ) pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node) result = gevent.event.AsyncResult() results = [] def message_received(message, headers): # Heads log.warn(' data received!') results.append(message) if len(results) >= len( data_product_stream_ids ) * message_count_per_stream: #Only wait for so many messages - per stream result.set(True) subscriber = subscriber_registrar.create_subscriber( exchange_name='workflow_test', callback=message_received) subscriber.start() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id) # Assert that we have received data assertions(result.get(timeout=30)) self.pubsubclient.deactivate_subscription( subscription_id=salinity_subscription_id) subscriber.stop() return results def _validate_messages(self, results): cc = self.container assertions = self.assertTrue first_salinity_values = None for message in results: try: psd = PointSupplementStreamParser( stream_definition=self.ctd_stream_def, stream_granule=message) temp = psd.get_values('temperature') log.info(psd.list_field_names()) except KeyError as ke: temp = None if temp is not None: assertions(isinstance(temp, numpy.ndarray)) log.info('temperature=' + str(numpy.nanmin(temp))) first_salinity_values = None else: psd = PointSupplementStreamParser( stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message) log.info(psd.list_field_names()) # Test the handy info method for the names of fields in the stream def assertions('salinity' in psd.list_field_names()) # you have to know the name of the coverage in stream def salinity = psd.get_values('salinity') log.info('salinity=' + str(numpy.nanmin(salinity))) assertions(isinstance(salinity, numpy.ndarray)) assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0 if first_salinity_values is None: first_salinity_values = salinity.tolist() else: second_salinity_values = salinity.tolist() assertions( len(first_salinity_values) == len( second_salinity_values)) for idx in range(0, len(first_salinity_values)): assertions(first_salinity_values[idx] * 2.0 == second_salinity_values[idx]) def _create_salinity_data_process_definition(self): # Salinity: Data Process Definition #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='ctd_salinity') if len(dpd) > 0: return dpd[0] log.debug("Create data process definition SalinityTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_salinity', description='create a salinity data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Excpetion as ex: self.fail( "failed to create new SalinityTransform data process definition: %s" % ex) # create a stream definition for the data from the salinity Transform sal_stream_def_id = self.pubsubclient.create_stream_definition( container=SalinityTransform.outgoing_stream_def, name='Salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition( sal_stream_def_id, ctd_L2_salinity_dprocdef_id) return ctd_L2_salinity_dprocdef_id def _create_salinity_doubler_data_process_definition(self): #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler') if len(dpd) > 0: return dpd[0] # Salinity Doubler: Data Process Definition log.debug("Create data process definition SalinityDoublerTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='salinity_doubler', description='create a salinity doubler data product', module='ion.processes.data.transforms.example_double_salinity', class_name='SalinityDoubler', process_source='SalinityDoubler source code here...') try: salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Exception as ex: self.fail( "failed to create new SalinityDoubler data process definition: %s" % ex) # create a stream definition for the data from the salinity Transform salinity_double_stream_def_id = self.pubsubclient.create_stream_definition( container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler') self.dataprocessclient.assign_stream_definition_to_data_process_definition( salinity_double_stream_def_id, salinity_doubler_dprocdef_id) return salinity_doubler_dprocdef_id def create_transform_process(self, data_process_definition_id, data_process_input_dp_id): data_process_definition = self.rrclient.read( data_process_definition_id) # Find the link between the output Stream Definition resource and the Data Process Definition resource stream_ids, _ = self.rrclient.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True) if not stream_ids: raise Inconsistent( "The data process definition %s is missing an association to an output stream definition" % data_process_definition._id) process_output_stream_def_id = stream_ids[0] #Concatenate the name of the workflow and data process definition for the name of the data product output data_process_name = data_process_definition.name # Create the output data product of the transform transform_dp_obj = IonObject( RT.DataProduct, name=data_process_name, description=data_process_definition.description) transform_dp_id = self.dataproductclient.create_data_product( transform_dp_obj, process_output_stream_def_id) self.dataproductclient.activate_data_product_persistence( data_product_id=transform_dp_id, persist_data=True, persist_metadata=True) #last one out of the for loop is the output product id output_data_product_id = transform_dp_id # Create the transform data process log.debug("create data_process and start it") data_process_id = self.dataprocessclient.create_data_process( data_process_definition._id, [data_process_input_dp_id], {'output': transform_dp_id}) self.dataprocessclient.activate_data_process(data_process_id) #Find the id of the output data stream stream_ids, _ = self.rrclient.find_objects(transform_dp_id, PRED.hasStream, None, True) if not stream_ids: raise Inconsistent( "The data process %s is missing an association to an output stream" % data_process_id) return data_process_id, output_data_product_id @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #@unittest.skip("Skipping for debugging ") def test_SA_transform_components(self): assertions = self.assertTrue #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) ### ### Setup the first transformation ### # Salinity: Data Process Definition ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition( ) l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process( ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product_id) ## get the stream id for the transform outputs stream_ids, _ = self.rrclient.find_objects( ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) sal_stream_id = stream_ids[0] data_product_stream_ids.append(sal_stream_id) ### ### Setup the second transformation ### # Salinity Doubler: Data Process Definition salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition( ) salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process( salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id) stream_ids, _ = self.rrclient.find_objects( salinity_doubler_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) sal_dbl_stream_id = stream_ids[0] data_product_stream_ids.append(sal_dbl_stream_id) #Start the input stream process ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id) #Start te output stream listener to monitor and verify messages results = self._start_output_stream_listener(data_product_stream_ids) #Stop the transform processes self.dataprocessclient.deactivate_data_process( salinity_double_data_process_id) self.dataprocessclient.deactivate_data_process( l2_salinity_all_data_process_id) # stop the flow parse the messages... self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Validate the data from each of the messages along the way self._validate_messages(results) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #@unittest.skip("Skipping for debugging ") def test_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Salinity_Test_Workflow', description='tests a workflow of multiple transform data processes' ) workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #Add a transformation process definition ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False ) #Don't persist the intermediate data product workflow_def_obj.workflow_steps.append(workflow_step_obj) #Add a transformation process definition salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=salinity_doubler_dprocdef_id, output_data_product_name=workflow_data_product_name) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 2) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, timeout=30) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Verify the output data product name matches what was specified in the workflow definition workflow_product = self.rrclient.read(workflow_product_id) assertions(workflow_product.name == workflow_data_product_name) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 2) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the input stream process ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id) #Start the output stream listener to monitor and verify messages results = self._start_output_stream_listener(data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id, False, timeout=15) # Should test true at some point #Make sure the Workflow object was removed objs, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(objs) == 0) # stop the flow parse the messages... self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Validate the data from each of the messages along the way self._validate_messages(results) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids, _ = self.rrclient.find_resources( restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0) def _create_google_dt_data_process_definition(self): #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform') if len(dpd) > 0: return dpd[0] # Data Process Definition log.debug("Create data process definition GoogleDtTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='google_dt_transform', description='Convert data streams to Google DataTables', module='ion.processes.data.transforms.viz.google_dt', class_name='VizTransformGoogleDT', process_source='VizTransformGoogleDT source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Exception as ex: self.fail( "failed to create new VizTransformGoogleDT data process definition: %s" % ex) # create a stream definition for the data from the stream_def_id = self.pubsubclient.create_stream_definition( container=VizTransformGoogleDT.outgoing_stream_def, name='VizTransformGoogleDT') self.dataprocessclient.assign_stream_definition_to_data_process_definition( stream_def_id, procdef_id) return procdef_id def _validate_google_dt_results(self, results_stream_def, results): cc = self.container assertions = self.assertTrue for g in results: if isinstance(g, Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) #log.warn(tx.pretty_print()) #log.warn(rdt.pretty_print()) gdt_component = rdt['google_dt_components'][0] assertions( gdt_component['viz_product_type'] == 'google_realtime_dt') gdt_description = gdt_component['data_table_description'] gdt_content = gdt_component['data_table_content'] assertions(gdt_description[0][0] == 'time') assertions(len(gdt_description) > 1) assertions(len(gdt_content) >= 0) return def _create_mpl_graphs_data_process_definition(self): #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform') if len(dpd) > 0: return dpd[0] #Data Process Definition log.debug("Create data process definition MatplotlibGraphsTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='mpl_graphs_transform', description='Convert data streams to Matplotlib graphs', module='ion.processes.data.transforms.viz.matplotlib_graphs', class_name='VizTransformMatplotlibGraphs', process_source='VizTransformMatplotlibGraphs source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Exception as ex: self.fail( "failed to create new VizTransformMatplotlibGraphs data process definition: %s" % ex) # create a stream definition for the data stream_def_id = self.pubsubclient.create_stream_definition( container=VizTransformMatplotlibGraphs.outgoing_stream_def, name='VizTransformMatplotlibGraphs') self.dataprocessclient.assign_stream_definition_to_data_process_definition( stream_def_id, procdef_id) return procdef_id def _validate_mpl_graphs_results(self, results_stream_def, results): cc = self.container assertions = self.assertTrue for g in results: if isinstance(g, Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) #log.warn(tx.pretty_print()) #log.warn(rdt.pretty_print()) graphs = rdt['matplotlib_graphs'] for graph in graphs: assertions( graph['viz_product_type'] == 'matplotlib_graphs') # check to see if the list (numpy array) contians actual images assertions( imghdr.what(graph['image_name'], graph['image_obj']) == 'png') return @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #unittest.skip("Skipping for debugging ") def test_google_dt_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='GoogleDT_Test_Workflow', description= 'Tests the workflow of converting stream data to Google DT') #Add a transformation process definition google_dt_procdef_id = self._create_google_dt_data_process_definition() workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the input stream process ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id) #Start the output stream listener to monitor and verify messages results = self._start_output_stream_listener(data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id, False) # Should test true at some point # stop the flow parse the messages... self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Validate the data from each of the messages along the way self._validate_google_dt_results( VizTransformGoogleDT.outgoing_stream_def, results) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids, _ = self.rrclient.find_resources( restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #@unittest.skip("Skipping for debugging ") def test_mpl_graphs_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Mpl_Graphs_Test_Workflow', description= 'Tests the workflow of converting stream data to Matplotlib graphs' ) #Add a transformation process definition mpl_graphs_procdef_id = self._create_mpl_graphs_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=mpl_graphs_procdef_id) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the input stream process ctd_sim_pid = self._start_sinusoidal_input_stream_process( ctd_stream_id) #Start the output stream listener to monitor and verify messages results = self._start_output_stream_listener(data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id, False) # Should test true at some point # stop the flow parse the messages... self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Validate the data from each of the messages along the way self._validate_mpl_graphs_results( VizTransformGoogleDT.outgoing_stream_def, results) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids, _ = self.rrclient.find_resources( restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #@unittest.skip("Skipping for debugging ") def test_multiple_workflow_instances(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Multiple_Test_Workflow', description='Tests the workflow of converting stream data') #Add a transformation process definition google_dt_procdef_id = self._create_google_dt_data_process_definition() workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the first input data product ctd_stream_id1, ctd_parsed_data_product_id1 = self._create_ctd_input_stream_and_data_product( 'ctd_parsed1') data_product_stream_ids.append(ctd_stream_id1) #Create and start the first workflow workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id1) #Create the second input data product ctd_stream_id2, ctd_parsed_data_product_id2 = self._create_ctd_input_stream_and_data_product( 'ctd_parsed2') data_product_stream_ids.append(ctd_stream_id2) #Create and start the first workflow workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id2) #Walk the associations to find the appropriate output data streams to validate the messages workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 2) #Start the first input stream process ctd_sim_pid1 = self._start_sinusoidal_input_stream_process( ctd_stream_id1) #Start the second input stream process ctd_sim_pid2 = self._start_simple_input_stream_process(ctd_stream_id2) #Start the output stream listener to monitor a set number of messages being sent through the workflows results = self._start_output_stream_listener( data_product_stream_ids, message_count_per_stream=5) # stop the flow of messages... self.process_dispatcher.cancel_process( ctd_sim_pid1 ) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid2) #Stop the first workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id1, False) # Should test true at some point #Stop the second workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id2, False) # Should test true at some point workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 0) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids, _ = self.rrclient.find_resources( restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0) aid_list = self.rrclient.find_associations( workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aid_list) == 0)
class TestObservatoryManagementFullIntegration(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient() self.RR2 = EnhancedResourceRegistryClient(self.RR) self.OMS = ObservatoryManagementServiceClient() self.org_management_service = OrgManagementServiceClient() self.IMS = InstrumentManagementServiceClient() self.dpclient = DataProductManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.dataset_management = DatasetManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.data_product_management = DataProductManagementServiceClient() self._load_stage = 0 self._resources = {} def preload_ooi(self, stage=STAGE_LOAD_ASSETS): # Preloads OOI up to a given stage if self._load_stage >= stage: return if self._load_stage < STAGE_LOAD_ORGS: log.info( "--------------------------------------------------------------------------------------------------------" ) log.info("Preloading stage: %s (OOIR2 Orgs, users, roles)", STAGE_LOAD_ORGS) # load_OOIR2_scenario self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="OOIR2", path="master", )) self._load_stage = STAGE_LOAD_ORGS if self._load_stage < STAGE_LOAD_PARAMS: log.info( "--------------------------------------------------------------------------------------------------------" ) log.info("Preloading stage: %s (BASE params, streamdefs)", STAGE_LOAD_PARAMS) # load_parameter_scenarios self.container.spawn_process( "Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories= "ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) self._load_stage = STAGE_LOAD_PARAMS if self._load_stage < STAGE_LOAD_AGENTS: log.info( "--------------------------------------------------------------------------------------------------------" ) log.info("Preloading stage: %s (OOIR2_I agents, model links)", STAGE_LOAD_AGENTS) # load_OOIR2_agents self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="OOIR2_I", path="master", )) self._load_stage = STAGE_LOAD_AGENTS if self._load_stage < STAGE_LOAD_ASSETS: log.info( "--------------------------------------------------------------------------------------------------------" ) log.info( "Preloading stage: %s (OOI assets linked to params, agents)", STAGE_LOAD_ASSETS) # load_ooi_assets self.container.spawn_process( "Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", loadooi="True", path="master", assets="res/preload/r2_ioc/ooi_assets", bulk="True", debug="True", ooiuntil="9/1/2013", ooiparams="True", #excludecategories: DataProduct,DataProductLink,Deployment,Workflow,WorkflowDefinition )) self._load_stage = STAGE_LOAD_ASSETS # 'DataProduct,DataProductLink,WorkflowDefinition,ExternalDataProvider,ExternalDatasetModel,ExternalDataset,ExternalDatasetAgent,ExternalDatasetAgentInstance', @unittest.skip('Work in progress') def test_observatory(self): self._load_stage = 0 self._resources = {} passing = True self.assertTrue(True) # LOAD STEP 1 self.preload_ooi(stage=STAGE_LOAD_ORGS) passing &= self.orguserrole_assertions() # LOAD STEP 2 self.preload_ooi(stage=STAGE_LOAD_PARAMS) passing &= self.parameter_assertions() # LOAD STEP 3 self.preload_ooi(stage=STAGE_LOAD_AGENTS) passing &= self.agent_assertions() # LOAD STEP 4 self.preload_ooi(stage=STAGE_LOAD_ASSETS) # Check OOI preloaded resources to see if they match needs for this test and for correctness passing &= self.sites_assertions() passing &= self.device_assertions() passing &= self.deployment_assertions() # Extensive tests on select RSN nodes passing &= self.rsn_node_checks() # Extensive tests on select RSN instruments passing &= self.check_rsn_instrument() passing &= self.check_rsn_instrument_data_product() # Extensive tests on a glider #passing &= self.check_glider() # Extensive tests on a CG assembly #passing &= self.check_cg_assembly() # Add a new instrument agent # Add a new instrument agent instance # Check DataProducts # Check Provenance IonIntegrationTestCase.assertTrue(self, passing) # ------------------------------------------------------------------------- def orguserrole_assertions(self): passing = True passing &= self._check_marine_facility("MF_CGSN") passing &= self._check_marine_facility("MF_RSN") passing &= self._check_marine_facility("MF_EA") return passing def _check_marine_facility(self, preload_id): passing = True log.debug("Checking marine facility %s and associations", preload_id) mf_obj = self.retrieve_ooi_asset(preload_id) mf_id = mf_obj._id self._resources[preload_id] = mf_id passing &= self.assertEquals(mf_obj.lcstate, LCS.DEPLOYED) res_list, _ = self.RR.find_objects(subject=mf_id, predicate=PRED.hasMembership, id_only=True) passing &= self.assertTrue(len(res_list) >= 3) res_list, _ = self.RR.find_objects(subject=mf_id, predicate=PRED.hasRole, id_only=False) passing &= self.assertTrue(len(res_list) >= 5) passing &= self._check_role_assignments(res_list, "ORG_MANAGER") passing &= self._check_role_assignments(res_list, "OBSERVATORY_OPERATOR") passing &= self._check_role_assignments(res_list, "INSTRUMENT_OPERATOR") return passing def _check_role_assignments(self, role_list, role_name): passing = True role_obj = self._find_resource_in_list(role_list, "governance_name", role_name) if role_obj: res_list = self.RR.find_subjects(predicate=PRED.hasRole, object=role_obj._id, id_only=True) passing &= self.assertTrue(len(res_list) >= 1) return passing def parameter_assertions(self): passing = True pctx_list, _ = self.RR.find_resources_ext(restype=RT.ParameterContext) passing &= self.assertTrue(len(pctx_list) >= 10) pdict_list, _ = self.RR.find_resources_ext( restype=RT.ParameterDictionary) passing &= self.assertTrue(len(pdict_list) >= 10) sdef_list, _ = self.RR.find_resources_ext(restype=RT.StreamDefinition) passing &= self.assertTrue(len(sdef_list) >= 10) # Verify that a PDict has the appropriate QC parameters defined pdicts, _ = self.RR.find_resources_ext(restype=RT.ParameterDictionary, alt_id_ns='PRE', alt_id='DICT110') passing &= self.assertTrue(len(pdicts) == 1) if not pdicts: return passing pdict = pdicts[0] # According to the latest SAF, density should NOT have trend parameters, _ = self.RR.find_objects(pdict, PRED.hasParameterContext) names = [i.name for i in parameters if i.name.startswith('density')] passing &= self.assertTrue('density_trndtst_qc' not in names) return passing def agent_assertions(self): passing = True # TODO: More tests? return passing def sites_assertions(self): passing = True observatory_list, _ = self.RR.find_resources_ext( restype=RT.Observatory) passing &= self.assertTrue(len(observatory_list) >= 40) for obs in observatory_list: passing &= self.assertEquals(obs.lcstate, LCS.DEPLOYED) platform_site_list, _ = self.RR.find_resources(RT.PlatformSite, id_only=False) log.debug('platform sites: %s', [ps.name for ps in platform_site_list]) passing &= self.assertTrue(len(platform_site_list) >= 30) return passing def device_assertions(self): passing = True platform_device_list, _ = self.RR.find_resources(RT.PlatformDevice, id_only=False) passing &= self.assertTrue(len(platform_device_list) >= 30) for pdev in platform_device_list: log.debug('platform device: %s', pdev.name) passing &= self.assertEquals(pdev.lcstate, LCS.PLANNED) platform_agent_list, _ = self.RR.find_resources(RT.PlatformAgent, id_only=False) passing &= self.assertTrue(len(platform_agent_list) >= 2) for pagent in platform_agent_list: log.debug('platform agent: %s', pagent.name) passing &= self.assertEquals(pagent.lcstate, LCS.DEPLOYED) instrument_agent_list, _ = self.RR.find_resources(RT.InstrumentAgent, id_only=False) passing &= self.assertTrue(len(instrument_agent_list) >= 3) for iagent in instrument_agent_list: log.debug('instrument agent: %s', iagent.name) passing &= self.assertEquals(iagent.lcstate, LCS.DEPLOYED) model_list, _ = self.RR.find_objects(subject=iagent._id, predicate=PRED.hasModel, id_only=True) passing &= self.assertTrue( len(model_list) >= 1, "IA %s" % iagent.name) return passing def deployment_assertions(self): passing = True deployment_list, _ = self.RR.find_resources(RT.Deployment, id_only=False) passing &= self.assertTrue(len(deployment_list) >= 30) for deploy in deployment_list: log.debug('deployment: %s', deploy.name) passing &= self.assertEquals(deploy.lcstate, LCS.DEPLOYED) return passing def rsn_node_checks(self): """ Current preload creates: - PlatformDevice in PLANNED - PlatformSite in DEPLOYED - Deployment in DEPLOYED - Deployment is NOT activated """ passing = True dp_obj = self.retrieve_ooi_asset("CE04OSHY-PN01C_DEP") passing &= self.assertEquals(dp_obj.lcstate, LCS.DEPLOYED) passing &= self.assertEquals(dp_obj.availability, AS.AVAILABLE) log.debug( 'test_observatory retrieve CE04OSHY-PN01C_DEP deployment: %s', dp_obj) # Check existing RSN node CE04OSHY-LV01C Deployment (PLANNED lcstate) CE04OSHY_LV01C_deployment = self.retrieve_ooi_asset( 'CE04OSHY-LV01C_DEP') passing &= self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, LCS.DEPLOYED) passing &= self.assertEquals(CE04OSHY_LV01C_deployment.availability, AS.AVAILABLE) #self.dump_deployment(CE04OSHY_LV01C_deployment._id) log.debug( 'test_observatory retrieve RSN node CE04OSHY-LV01C Deployment: %s', CE04OSHY_LV01C_deployment) CE04OSHY_LV01C_device = self.retrieve_ooi_asset('CE04OSHY-LV01C_PD') # Set CE04OSHY-LV01C device to DEVELOPED state passing &= self.transition_lcs_then_verify( resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.DEVELOP, verify=LCS.DEVELOPED) # Set CE04OSHY-LV01C device to INTEGRATED state passing &= self.transition_lcs_then_verify( resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.INTEGRATE, verify=LCS.INTEGRATED) # Set CE04OSHY-LV01C device to DEPLOYED state passing &= self.transition_lcs_then_verify( resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.DEPLOY, verify=LCS.DEPLOYED) # Set CE04OSHY-LV01C Deployment to DEPLOYED state # NOTE: Deployments are created in DEPLOYED state, currently #self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_deployment._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Activate Deployment for CE04OSHY-LV01C self.OMS.activate_deployment(CE04OSHY_LV01C_deployment._id) log.debug( '--------- activate_deployment CE04OSHY_LV01C_deployment -------------- ' ) self.dump_deployment(CE04OSHY_LV01C_deployment._id) passing &= self.validate_deployment_activated( CE04OSHY_LV01C_deployment._id) # (optional) Start CE04OSHY-LV01C platform agent with simulator # NOTE: DataProduct is generated in DEPLOYED state # # Set DataProduct for CE04OSHY-LV01C platform to DEPLOYED state # output_data_product_ids, assns =self.RR.find_objects(subject=CE04OSHY_LV01C_device._id, predicate=PRED.hasOutputProduct, id_only=True) # if output_data_product_ids: # #self.assertEquals(len(child_devs), 3) # for output_data_product_id in output_data_product_ids: # log.debug('DataProduct for CE04OSHY-LV01C platform: %s', output_data_product_id) # self.transition_lcs_then_verify(resource_id=output_data_product_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Check events for CE04OSHY-LV01C platform # Check existing CE04OSBP-LJ01C Deployment (PLANNED lcstate) # dp_list, _ = self.RR.find_resources_ext(alt_id_ns="PRE", alt_id="CE04OSBP-LJ01C_DEP") # self.assertEquals(len(dp_list), 1) # CE04OSHY_LV01C_deployment = dp_list[0] # self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, 'PLANNED') # log.debug('test_observatory retrieve RSN node CE04OSBP-LJ01C Deployment: %s', CE04OSHY_LV01C_deployment) # Set CE04OSBP-LJ01C Deployment to DEPLOYED state # Update description and other attributes for CE04OSBP-LJ01C device resource # Create attachment (JPG image) for CE04OSBP-LJ01C device resource # Activate Deployment for CE04OSBP-LJ01C # (optional) Add/register CE04OSBP-LJ01C platform agent to parent agent # (optional) Start CE04OSBP-LJ01C platform agent return passing def check_rsn_instrument(self): """ Check existing RSN instrument CE04OSBP-LJ01C-06-CTDBPO108 Deployment (PLANNED lcstate) Current preload creates: - InstrumentDevice in PLANNED - InstrumentSite in DEPLOYED - Deployment in DEPLOYED - Deployment is activated """ passing = True CE04OSBP_LJ01C_06_CTDBPO108_deploy = self.retrieve_ooi_asset( 'CE04OSBP-LJ01C-06-CTDBPO108_DEP') self.dump_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) #passing &= self.assertEquals(CE04OSBP_LJ01C_06_CTDBPO108_deploy.lcstate, 'PLANNED') # Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEVELOPED state CE04OSBP_LJ01C_06_CTDBPO108_device = self.retrieve_ooi_asset( 'CE04OSBP-LJ01C-06-CTDBPO108_ID') passing &= self.transition_lcs_then_verify( resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEVELOP, verify='DEVELOPED') # Set CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state passing &= self.transition_lcs_then_verify( resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED') # Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state passing &= self.transition_lcs_then_verify( resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Set CE04OSBP-LJ01C-06-CTDBPO108 Deployment to DEPLOYED state #self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Activate Deployment for CE04OSBP-LJ01C-06-CTDBPO108 instrument log.debug( '--------- activate_deployment CE04OSBP-LJ01C-06-CTDBPO108 deployment -------------- ' ) self.OMS.activate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) passing &= self.validate_deployment_activated( CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) # (optional) Add/register CE04OSBP-LJ01C-06-CTDBPO108 instrument agent to parent agent # (optional) Start CE04OSBP-LJ01C-06-CTDBPO108 instrument agent with simulator # Set all DataProducts for CE04OSBP-LJ01C-06-CTDBPO108 to DEPLOYED state # (optional) Create a substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device CE04OSBP_LJ01C_06_CTDBPO108_isite = self.retrieve_ooi_asset( 'CE04OSBP-LJ01C-06-CTDBPO108') ## create device here: retrieve CTD Mooring on Mooring Riser 001 - similiar? GP03FLMB_RI001_10_CTDMOG999_ID_idevice = self.retrieve_ooi_asset( 'GP03FLMB-RI001-10-CTDMOG999_ID') deploy_id_2 = self.create_basic_deployment( name='CE04OSBP-LJ01C-06-CTDBPO108_DEP2', description= 'substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device' ) self.OMS.assign_device_to_deployment( instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_2) self.OMS.assign_site_to_deployment( instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_2) self.dump_deployment(deploy_id_2) # (optional) Activate this second deployment - check first deployment is deactivated self.OMS.deactivate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) passing &= self.validate_deployment_deactivated( CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) # log.debug('Activate deployment deploy_id_2') # self.get_deployment_ids(deploy_id_2) # self.dump_deployment(deploy_id_2, "deploy_id_2") # self.OMS.activate_deployment(deploy_id_2) # passing &= self.validate_deployment_deactivated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) # # # (optional) Set first CE04OSBP-LJ01C-06-CTDBPO108 Deployment to INTEGRATED state # passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED') # # # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state # passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED') # # # # (optional) Create a third Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a same device from first deployment # deploy_id_3 = self.create_basic_deployment(name='CE04OSBP-LJ01C-06-CTDBPO108_DEP3', description='substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with same device as first') # self.IMS.deploy_instrument_device(instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_3) # self.OMS.deploy_instrument_site(instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_3) # self.dump_deployment(deploy_id_3) # # # # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state # passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # # # (optional) Activate this third deployment - check second deployment is deactivated # log.debug('Activate deployment deploy_id_3') # self.dump_deployment(deploy_id_3) # self.OMS.activate_deployment(deploy_id_3) # #todo: check second deployment is deactivated return passing def check_data_product_reference(self, reference_designator, output=[]): passing = True data_product_ids, _ = self.RR.find_resources_ext( alt_id_ns='PRE', alt_id='%s_DPI1' % reference_designator, id_only=True) # Assuming DPI1 is parsed passing &= self.assertEquals(len(data_product_ids), 1) if not data_product_ids: return passing # Let's go ahead and activate it data_product_id = data_product_ids[0] self.dpclient.activate_data_product_persistence(data_product_id) self.addCleanup(self.dpclient.suspend_data_product_persistence, data_product_id) dataset_ids, _ = self.RR.find_objects(data_product_id, PRED.hasDataset, id_only=True) passing &= self.assertEquals(len(dataset_ids), 1) if not dataset_ids: return passing dataset_id = dataset_ids[0] stream_def_ids, _ = self.RR.find_objects(data_product_id, PRED.hasStreamDefinition, id_only=True) passing &= self.assertEquals(len(dataset_ids), 1) if not stream_def_ids: return passing stream_def_id = stream_def_ids[0] output.append((data_product_id, stream_def_id, dataset_id)) return passing def check_tempsf_instrument_data_product(self, reference_designator): passing = True info_list = [] passing &= self.check_data_product_reference(reference_designator, info_list) if not passing: return passing data_product_id, stream_def_id, dataset_id = info_list.pop() now = time.time() ntp_now = now + 2208988800 rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [ntp_now] rdt['temperature'] = [[ 25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873, 15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993, 10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098, 5.3456, 4.2994, 4.3009 ]] dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt) passing &= self.assertTrue(dataset_monitor.wait()) if not passing: return passing granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now]) passing &= self.assert_array_almost_equal(rdt['temperature'], [[ 25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873, 15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993, 10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098, 5.3456, 4.2994, 4.3009 ]]) return passing def check_trhph_instrument_data_products(self, reference_designator): passing = True info_list = [] passing &= self.check_data_product_reference(reference_designator, info_list) if not passing: return passing data_product_id, stream_def_id, dataset_id = info_list.pop() pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary( stream_def_id) passing &= self.assertEquals(pdict.name, 'trhph_sample') rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) # calibration constants a = 1.98e-9 b = -2.45e-6 c = 9.28e-4 d = -0.0888 e = 0.731 V_s = 1.506 V_c = 0. T = 11.8 r1 = 0.906 r2 = 4.095 r3 = 4.095 ORP_V = 1.806 Cl = np.nan offset = 2008 gain = 4.0 # Normally this would be 50 per the DPS but the precision is %4.0f which truncates the values to the nearest 1... ORP = ((ORP_V * 1000.) - offset) / gain ntp_now = time.time() + 2208988800 rdt['cc_a'] = [a] rdt['cc_b'] = [b] rdt['cc_c'] = [c] rdt['cc_d'] = [d] rdt['cc_e'] = [e] rdt['ref_temp_volts'] = [V_s] rdt['resistivity_temp_volts'] = [V_c] rdt['eh_sensor'] = [ORP_V] rdt['resistivity_5'] = [r1] rdt['resistivity_x1'] = [r2] rdt['resistivity_x5'] = [r3] rdt['cc_offset'] = [offset] rdt['cc_gain'] = [gain] rdt['time'] = [ntp_now] passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'], [T], 2) passing &= self.assert_array_almost_equal( rdt['vent_fluid_chloride_conc'], [Cl], 4) passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP], 4) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt) passing &= self.assertTrue(dataset_monitor.wait()) if not passing: return passing granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'], [T], 2) passing &= self.assert_array_almost_equal( rdt['vent_fluid_chloride_conc'], [Cl], 4) passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP], 4) return passing def check_vel3d_instrument_data_products(self, reference_designator): passing = True info_list = [] passing &= self.check_data_product_reference(reference_designator, info_list) if not passing: return passing data_product_id, stream_def_id, dataset_id = info_list.pop() pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary( stream_def_id) self.assertEquals(pdict.name, 'vel3d_b_sample') rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) lat = 14.6846 lon = -51.044 ts = np.array([ 3319563600, 3319567200, 3319570800, 3319574400, 3319578000, 3319581600, 3319585200, 3319588800, 3319592400, 3319596000 ], dtype=np.float) ve = np.array([-3.2, 0.1, 0., 2.3, -0.1, 5.6, 5.1, 5.8, 8.8, 10.3]) vn = np.array([18.2, 9.9, 12., 6.6, 7.4, 3.4, -2.6, 0.2, -1.5, 4.1]) vu = np.array([-1.1, -0.6, -1.4, -2, -1.7, -2, 1.3, -1.6, -1.1, -4.5]) ve_expected = np.array([ -0.085136, -0.028752, -0.036007, 0.002136, -0.023158, 0.043218, 0.056451, 0.054727, 0.088446, 0.085952 ]) vn_expected = np.array([ 0.164012, 0.094738, 0.114471, 0.06986, 0.07029, 0.049237, -0.009499, 0.019311, 0.012096, 0.070017 ]) vu_expected = np.array([ -0.011, -0.006, -0.014, -0.02, -0.017, -0.02, 0.013, -0.016, -0.011, -0.045 ]) rdt['time'] = ts rdt['lat'] = [lat] * 10 rdt['lon'] = [lon] * 10 rdt['turbulent_velocity_east'] = ve rdt['turbulent_velocity_north'] = vn rdt['turbulent_velocity_up'] = vu passing &= self.assert_array_almost_equal( rdt['eastward_turbulent_velocity'], ve_expected) passing &= self.assert_array_almost_equal( rdt['northward_turbulent_velocity'], vn_expected) passing &= self.assert_array_almost_equal( rdt['upward_turbulent_velocity'], vu_expected) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt) passing &= self.assertTrue(dataset_monitor.wait()) if not passing: return passing granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) passing &= self.assert_array_almost_equal( rdt['eastward_turbulent_velocity'], ve_expected) passing &= self.assert_array_almost_equal( rdt['northward_turbulent_velocity'], vn_expected) passing &= self.assert_array_almost_equal( rdt['upward_turbulent_velocity'], vu_expected) return passing def check_presta_instrument_data_products(self, reference_designator): # Check the parsed data product make sure it's got everything it needs and can be published persisted etc. # Absolute Pressure (SFLPRES_L0) is what comes off the instrumnet, SFLPRES_L1 is a pfunc # Let's go ahead and publish some fake data!!! # According to https://alfresco.oceanobservatories.org/alfresco/d/d/workspace/SpacesStore/63e16865-9d9e-4b11-b0b3-d5658faa5080/1341-00230_Data_Product_Spec_SFLPRES_OOI.pdf # Appendix A. Example 1. # p_psia_tide = 14.8670 # the tide should be 10.2504 passing = True info_list = [] passing &= self.check_data_product_reference(reference_designator, info_list) if not passing: return passing data_product_id, stream_def_id, dataset_id = info_list.pop() now = time.time() ntp_now = now + 2208988800. rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [ntp_now] rdt['absolute_pressure'] = [14.8670] passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue( dataset_monitor.wait()) # Bumped to 20 to keep buildbot happy if not passing: return passing granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now]) passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4) passing &= self.assert_array_almost_equal(rdt['absolute_pressure'], [14.8670], 4) return passing def check_rsn_instrument_data_product(self): passing = True # for RS03AXBS-MJ03A-06-PRESTA301 (PREST-A) there are a few listed data products # Parsed, Engineering # SFLPRES-0 SFLPRES-1 # Check for the two data products and make sure they have the proper parameters # SFLPRES-0 should data_products, _ = self.RR.find_resources_ext( alt_id_ns='PRE', alt_id='RS03AXBS-MJ03A-06-PRESTA301_SFLPRES_L0_DPID', id_only=True) passing &= self.assertTrue(len(data_products) == 1) if not data_products: return passing data_product_id = data_products[0] stream_defs, _ = self.RR.find_objects(data_product_id, PRED.hasStreamDefinition, id_only=False) passing &= self.assertTrue(len(stream_defs) == 1) if not stream_defs: return passing # Assert that the stream definition has the correct reference designator stream_def = stream_defs[0] passing &= self.assertEquals( stream_def.stream_configuration['reference_designator'], 'RS03AXBS-MJ03A-06-PRESTA301') # Get the pdict and make sure that the parameters corresponding to the available fields # begin with the appropriate data product identifier pdict_ids, _ = self.RR.find_objects(stream_def, PRED.hasParameterDictionary, id_only=True) passing &= self.assertEquals(len(pdict_ids), 1) if not pdict_ids: return passing pdict_id = pdict_ids[0] pdict = DatasetManagementService.get_parameter_dictionary(pdict_id) available_params = [ pdict.get_context(i) for i in pdict.keys() if i in stream_def.available_fields ] for p in available_params: if p.name == 'time': # Ignore the domain parameter continue passing &= self.assertTrue(p.ooi_short_name.startswith('SFLPRES')) passing &= self.check_presta_instrument_data_products( 'RS01SLBS-MJ01A-06-PRESTA101') passing &= self.check_vel3d_instrument_data_products( 'RS01SLBS-MJ01A-12-VEL3DB101') passing &= self.check_presta_instrument_data_products( 'RS03AXBS-MJ03A-06-PRESTA301') passing &= self.check_vel3d_instrument_data_products( 'RS03AXBS-MJ03A-12-VEL3DB301') passing &= self.check_tempsf_instrument_data_product( 'RS03ASHS-MJ03B-07-TMPSFA301') passing &= self.check_vel3d_instrument_data_products( 'RS03INT2-MJ03D-12-VEL3DB304') passing &= self.check_trhph_instrument_data_products( 'RS03INT1-MJ03C-10-TRHPHA301') self.data_product_management.activate_data_product_persistence( data_product_id) dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset( data_product_id) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4) self.assert_array_almost_equal(rdt['absolute_pressure'], [14.8670], 4) self.data_product_management.suspend_data_product_persistence( data_product_id) # Should do nothing and not raise anything return passing def check_glider(self): ''' # Check that glider GP05MOAS-GL001 assembly is defined by OOI preload (3 instruments) ''' passing = True GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD') child_devs, assns = self.RR.find_objects( subject=GP05MOAS_GL001_device._id, predicate=PRED.hasDevice, id_only=True) passing &= self.assertEquals(len(child_devs), 3) # Set GP05MOAS-GL001 Deployment to DEPLOYED GP05MOAS_GL001_deploy = self.retrieve_ooi_asset('GP05MOAS-GL001_DEP') passing &= self.transition_lcs_then_verify( resource_id=GP05MOAS_GL001_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Activate Deployment for GP05MOAS-GL001 #self.OMS.activate_deployment(GP05MOAS_GL001_deploy._id) # Deactivate Deployment for GP05MOAS-GL001 #self.OMS.deactivate_deployment(GP05MOAS_GL001_deploy._id) # Create a new Deployment resource X without any assignment x_deploy_id = self.create_basic_deployment( name='X_Deployment', description='new Deployment resource X without any assignment') # Assign Deployment X to site GP05MOAS-GL001 GP05MOAS_GL001_psite = self.retrieve_ooi_asset('GP05MOAS-GL001') self.OMS.assign_site_to_deployment(GP05MOAS_GL001_psite._id, x_deploy_id) # Assign Deployment X to first device for GP05MOAS-GL001 GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD') self.OMS.assign_device_to_deployment(GP05MOAS_GL001_device._id, x_deploy_id) # Set GP05MOAS-GL001 Deployment to PLANNED state #self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.PLAN, verify='PLANNED') # ??? already in planned # Set second GP05MOAS-GL001 Deployment to DEPLOYED passing &= self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') self.dump_deployment(x_deploy_id) # Activate second Deployment for GP05MOAS-GL001 #self.OMS.activate_deployment(x_deploy_id) # Deactivate second Deployment for GP05MOAS-GL001 #self.OMS.deactivate_deployment(x_deploy_id) return passing def check_cg_assembly(self): passing = True # Set several CE01ISSM-RI002-* instrument devices to DEVELOPED state # Assemble several CE01ISSM-RI002-* instruments to a CG CE01ISSM-RI002 component platform # Set several CE01ISSM-RI002-* instrument devices to INTEGRATED state # Assemble CE01ISSM-RI002 platform to CG CE01ISSM-LM001 station platform # Set CE01ISSM-RI002 component device to INTEGRATED state # Set CE01ISSM-LM001 station device to INTEGRATED state # Set CE01ISSM-LM001 station device to DEPLOYED state (children maybe too?) # Set CE01ISSM-LM001 Deployment to DEPLOYED # Activate CE01ISSM-LM001 platform assembly deployment # Dectivate CE01ISSM-LM001 platform assembly deployment # Set CE01ISSM-LM001 Deployment to INTEGRATED state # Set CE01ISSM-LM001 station device to INTEGRATED state # Set CE01ISSM-RI002 component device to INTEGRATED state # Set CE01ISSM-RI002 component device to INTEGRATED state # Disassemble CE01ISSM-RI002 platform from CG CE01ISSM-LM001 station platform # Disassemble all CE01ISSM-RI002-* instruments from a CG CE01ISSM-RI002 component platform # Retire instrument one for CE01ISSM-RI002-* # Retire device one for CE01ISSM-RI002 # Retire device one for CE01ISSM-LM001 return passing # ------------------------------------------------------------------------- def retrieve_ooi_asset(self, alt_id='', namespace='PRE'): dp_list, _ = self.RR.find_resources_ext(alt_id_ns=namespace, alt_id=alt_id) self.assertEquals(len(dp_list), 1) return dp_list[0] def transition_lcs_then_verify(self, resource_id, new_lcs_state, verify): ret = self.RR2.advance_lcs(resource_id, new_lcs_state) resource_obj = self.RR.read(resource_id) return self.assertEquals(resource_obj.lcstate, verify) def create_basic_deployment(self, name='', description=''): start = IonTime(datetime.datetime(2013, 1, 1)) end = IonTime(datetime.datetime(2014, 1, 1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name=name, description=description, context=IonObject( OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) return self.OMS.create_deployment(deployment_obj) def validate_deployment_activated(self, deployment_id=''): site_id, device_id = self.get_deployment_ids(deployment_id) assocs = self.RR.find_associations(subject=site_id, predicate=PRED.hasDevice, object=device_id) return self.assertEquals(len(assocs), 1) def validate_deployment_deactivated(self, deployment_id=''): site_id, device_id = self.get_deployment_ids(deployment_id) assocs = self.RR.find_associations(subject=site_id, predicate=PRED.hasDevice, object=device_id) return self.assertEquals(len(assocs), 0) def dump_deployment(self, deployment_id='', name=""): #site_id, device_id = self.get_deployment_ids(deployment_id) resource_list, _ = self.RR.find_subjects(predicate=PRED.hasDeployment, object=deployment_id, id_only=True) resource_list.append(deployment_id) resources = self.RR.read_mult(resource_list) log.debug('--------- dump_deployment %s summary---------------', name) for resource in resources: log.debug('%s: %s (%s)', resource._get_type(), resource.name, resource._id) log.debug('--------- dump_deployment %s full dump ---------------', name) for resource in resources: log.debug('resource: %s ', resource) log.debug('--------- dump_deployment %s end ---------------', name) #assocs = self.container.resource_registry.find_assoctiations(anyside=deployment_id) # assocs = Container.instance.resource_registry.find_assoctiations(anyside=deployment_id) # log.debug('--------- dump_deployment ---------------') # for assoc in assocs: # log.debug('SUBJECT: %s PREDICATE: %s OBJET: %s', assoc.s, assoc.p, assoc.o) # log.debug('--------- dump_deployment end ---------------') def get_deployment_ids(self, deployment_id=''): devices = [] sites = [] idevice_list, _ = self.RR.find_subjects(RT.InstrumentDevice, PRED.hasDeployment, deployment_id, id_only=True) pdevice_list, _ = self.RR.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, id_only=True) devices = idevice_list + pdevice_list self.assertEquals(1, len(devices)) isite_list, _ = self.RR.find_subjects(RT.InstrumentSite, PRED.hasDeployment, deployment_id, id_only=True) psite_list, _ = self.RR.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, id_only=True) sites = isite_list + psite_list self.assertEquals(1, len(sites)) return sites[0], devices[0] def _find_resource_in_list(self, res_list, attr, attr_val, assert_found=True): for res in res_list: v = getattr(res, attr, None) if v == attr_val: return res if assert_found: self.assertTrue( False, "Attribute %s value %s not found in list" % (attr, attr_val)) return None # ------------------------------------------------------------------------- def _get_caller(self): s = inspect.stack() return "%s:%s" % (s[2][1], s[2][2]) @assertion_wrapper def assert_array_almost_equal(self, *args, **kwargs): np.testing.assert_array_almost_equal(*args, **kwargs) @assertion_wrapper def assertEquals(self, *args, **kwargs): IonIntegrationTestCase.assertEquals(self, *args, **kwargs) @assertion_wrapper def assertTrue(self, *args, **kwargs): IonIntegrationTestCase.assertTrue(self, *args, **kwargs)
class IndexManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.ims_cli = IndexManagementServiceClient() self.rr_cli = ResourceRegistryServiceClient() self.index_name = 'test_index' def test_create_datastore_index(self): index_name = self.index_name ims_cli = self.ims_cli rr_cli = self.rr_cli options = SearchOptions() options.attribute_match = ['test_field'] index_id = ims_cli.create_index( name=index_name, content_type=IndexManagementService.DATASTORE_INDEX, options=options) index_result = self.rr_cli.read(index_id) self.assertIsInstance(index_result, ElasticSearchIndex) self.assertTrue(index_result.name == index_name) #====================================== # Clean up #====================================== rr_cli.delete(index_id) def test_read_index(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name index_res = Index(name=index_name) index_id, _ = rr_cli.create(index_res) index = ims_cli.read_index(index_id) self.assertIsInstance(index, Index) self.assertTrue(index.name == index_name) rr_cli.delete(index_id) def test_delete_index(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name index_res = Index(name=index_name) index_id, _ = rr_cli.create(index_res) ims_cli.delete_index(index_id) with self.assertRaises(NotFound): rr_cli.delete(index_id) def test_update_index(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name index_res = Index(name=index_name) index_id, _ = rr_cli.create(index_res) index = ims_cli.read_index(index_id) index.name = 'another' ims_cli.update_index(index) index = rr_cli.read(index_id) self.assertTrue(index.name == 'another') def test_find_indexes(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name #====================================== # Index Pool #====================================== indexes = [ Index(name='first'), Index(name='second'), Index(name='third') ] id_pool = list() for index in indexes: id_pool.append(rr_cli.create(index)[0]) index_id = ims_cli.find_indexes(index_name='second') index = ims_cli.read_index(index_id) self.assertTrue(index.name == 'second') #====================================== # Clean up #====================================== for index_id in id_pool: rr_cli.delete(index_id) def test_create_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli with self.assertRaises(BadRequest): ims_cli.create_collection('failing_collection') resources = [Resource(), Resource(), Resource()] resources = [rr_cli.create(i)[0] for i in resources] collection_id = ims_cli.create_collection('working_collection', resources) collection = rr_cli.read(collection_id) collection_resources = ims_cli.list_collection_resources(collection_id, id_only=True) self.assertTrue( set(collection_resources) == set(resources), '%s != %s' % (set(collection_resources), set(resources))) def test_read_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli collection = Collection(name='working_collection') collection_id, _ = rr_cli.create(collection) collection = ims_cli.read_collection(collection_id) self.assertTrue(collection.name == 'working_collection') def test_update_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli collection = Collection(name='useful_collection') collection_id, _ = rr_cli.create(collection) collection = rr_cli.read(collection_id) collection.name = 'nub' ims_cli.update_collection(collection) collection = rr_cli.read(collection_id) self.assertTrue(collection.name == 'nub') def test_delete_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli res = Resource() res_id, rev = rr_cli.create(res) collection_id = ims_cli.create_collection(name='test_collection', resources=[res_id]) ims_cli.delete_collection(collection_id) with self.assertRaises(NotFound): rr_cli.read(collection_id) def test_list_collection_resources(self): ims_cli = self.ims_cli rr_cli = self.rr_cli #======================================== # Resource Pool #======================================== resources = [ InformationResource(name='bean_counter'), InformationResource(name='lunar_rock'), InformationResource('aperature'), InformationResource('lemons') ] resources = [rr_cli.create(i)[0] for i in resources] collection = Collection(name='park_bench') collection_id = ims_cli.create_collection(name='park_bench', resources=resources) retval = ims_cli.list_collection_resources(collection_id, id_only=True) retval.sort() resources.sort() self.assertTrue(retval == resources, '%s != %s' % (retval, resources)) def test_find_collection(self): res_id, _ = self.rr_cli.create(Resource(name='test_res')) collection_id = self.ims_cli.create_collection('test', [res_id]) retval = self.ims_cli.find_collection(collection_name='test') self.assertTrue(retval[0] == collection_id) retval = self.ims_cli.find_collection(resource_ids=[res_id]) self.assertTrue(retval[0] == collection_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient( node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create( any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite)) platform_agent_instance_id, _ = self.RR.create( any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain=tdom, spatial_domain=sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension( instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Lifecycle self.assertEquals(len(extended_instrument.lcstate_transitions), 5) self.assertEquals( set(extended_instrument.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate'])) self.assertEquals(len(extended_instrument.availability_transitions), 2) self.assertEquals( set(extended_instrument.availability_transitions.keys()), set(['enable', 'announce'])) # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance( extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension( platform_device_id) self.assertEqual(1, len(extended_platform.portals)) self.assertEqual(1, len(extended_platform.portal_instruments)) #self.assertEqual(1, len(extended_platform.computed.portal_status.value)) # no agent started so NO statuses reported self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) self.assertEquals(extended_platform.platform_agent._id, platform_agent_id) self.assertEquals(len(extended_platform.lcstate_transitions), 5) self.assertEquals( set(extended_platform.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate'])) self.assertEquals(len(extended_platform.availability_transitions), 2) self.assertEquals( set(extended_platform.availability_transitions.keys()), set(['enable', 'announce'])) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual( ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue('Parsed_Canonical' in extended_instrument.computed. data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue('temp' in extended_instrument.computed. data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(DeviceStatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance( platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.OMS.force_delete_instrument_site(instrument_site_id) self.OMS.force_delete_platform_site(platform_site_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instModel_obj = IonObject(OT.CustomAttribute, name='SBE37IMModelAttr', description="model custom attr") instrument_model_id, _ = self.RR.create( any_old(RT.InstrumentModel, {"custom_attributes": [instModel_obj]})) instrument_device_id, _ = self.RR.create( any_old( RT.InstrumentDevice, { "custom_attributes": { "favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device( instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_data_producer(self): idevice_id = self.IMS.create_instrument_device( any_old(RT.InstrumentDevice)) self.assertEqual( 1, len( self.RR2. find_data_producer_ids_of_instrument_device_using_has_data_producer( idevice_id))) pdevice_id = self.IMS.create_platform_device(any_old( RT.PlatformDevice)) self.assertEqual( 1, len( self.RR2. find_data_producer_ids_of_platform_device_using_has_data_producer( pdevice_id))) @attr('PREP') def test_prepare_resource_support(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create( any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create( any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) instrument_device2_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_device3_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_device2_id, _ = self.RR.create(any_old(RT.PlatformDevice)) sensor_device2_id, _ = self.RR.create(any_old(RT.SensorDevice)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device2_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_device2_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device2_id, PRED.hasDevice, instrument_device2_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(sensor_device2_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device2_id, PRED.hasDevice, instrument_device2_id) sensor_model_id #is only a target #set lcstate - used for testing prepare - not setting all to DEVELOP, only some self.RR.execute_lifecycle_transition(instrument_agent_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_agent_id, LCE.DEVELOP) #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain=tdom, spatial_domain=sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") def ion_object_encoder(obj): return obj.__dict__ #First call to create instrument_data = self.IMS.prepare_instrument_device_support() #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, '') self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual( len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual( instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. resources), 1) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. resources[0]._id, instrument_agent_instance_id) self.assertEqual( len(instrument_data.associations['InstrumentModel']. associated_resources), 0) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. associated_resources), 0) self.assertEqual( len(instrument_data.associations['SensorDevice'].resources), 0) #Next call to update instrument_data = self.IMS.prepare_instrument_device_support( instrument_device_id) #print 'Update results' #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, instrument_device_id) self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual( len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual( instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. resources), 1) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. resources[0]._id, instrument_agent_instance_id) self.assertEqual( len(instrument_data.associations['InstrumentModel']. associated_resources), 1) self.assertEqual( instrument_data.associations['InstrumentModel']. associated_resources[0].s, instrument_device_id) self.assertEqual( instrument_data.associations['InstrumentModel']. associated_resources[0].o, instrument_model_id) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. associated_resources), 1) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. associated_resources[0].o, instrument_agent_instance_id) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. associated_resources[0].s, instrument_device_id) self.assertEqual( len(instrument_data.associations['SensorDevice'].resources), 1) self.assertEqual( instrument_data.associations['SensorDevice'].resources[0]._id, sensor_device_id) self.assertEqual( len(instrument_data.associations['SensorDevice']. associated_resources), 1) self.assertEqual( instrument_data.associations['SensorDevice']. associated_resources[0].o, instrument_device_id) self.assertEqual( instrument_data.associations['SensorDevice']. associated_resources[0].s, sensor_device_id) self.assertEqual( instrument_data.associations['InstrumentModel'].assign_request. request_parameters['instrument_device_id'], instrument_device_id) #test prepare for create of instrument agent instance instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support( ) #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, '') self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. resources), 2) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent'].resources ), 1) self.assertEqual( instrument_agent_data.associations['InstrumentAgent'].resources[0]. _id, instrument_agent_id) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. associated_resources), 0) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent']. associated_resources), 0) #test prepare for update of instrument agent instance to see if it is associated with the instrument that was created instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support( instrument_agent_instance_id=instrument_agent_instance_id) #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, instrument_agent_instance_id) self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. resources), 3) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent'].resources ), 1) self.assertEqual( instrument_agent_data.associations['InstrumentAgent'].resources[0]. _id, instrument_agent_id) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. associated_resources), 1) self.assertEqual( instrument_agent_data.associations['InstrumentDevice']. associated_resources[0].s, instrument_device_id) self.assertEqual( instrument_agent_data.associations['InstrumentDevice']. associated_resources[0].o, instrument_agent_instance_id) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent']. associated_resources), 1) self.assertEqual( instrument_agent_data.associations['InstrumentAgent']. associated_resources[0].o, instrument_agent_id) self.assertEqual( instrument_agent_data.associations['InstrumentAgent']. associated_resources[0].s, instrument_agent_instance_id) self.assertEqual( instrument_agent_data.associations['InstrumentAgent']. assign_request.request_parameters['instrument_agent_instance_id'], instrument_agent_instance_id) #test prepare for update of data product to see if it is associated with the instrument that was created data_product_data = self.DP.prepare_data_product_support( data_product_id1) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent=2) self.assertEqual(data_product_data._id, data_product_id1) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual( len(data_product_data.associations['StreamDefinition'].resources), 1) self.assertEqual( len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual( len(data_product_data.associations['StreamDefinition']. associated_resources), 1) self.assertEqual( data_product_data.associations['StreamDefinition']. associated_resources[0].s, data_product_id1) self.assertEqual( len(data_product_data.associations['Dataset'].associated_resources ), 0) self.assertEqual( len(data_product_data. associations['InstrumentDeviceHasOutputProduct'].resources), 3) self.assertEqual( len(data_product_data.associations[ 'InstrumentDeviceHasOutputProduct'].associated_resources), 1) self.assertEqual( data_product_data.associations['InstrumentDeviceHasOutputProduct']. associated_resources[0].s, instrument_device_id) self.assertEqual( data_product_data.associations['InstrumentDeviceHasOutputProduct']. associated_resources[0].o, data_product_id1) self.assertEqual( len(data_product_data.associations['PlatformDevice'].resources), 2) platform_data = self.IMS.prepare_platform_device_support() #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, '') self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual( len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual( platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual( len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual( platform_data.associations['PlatformAgentInstance'].resources[0]. _id, platform_agent_instance_id) self.assertEqual( len(platform_data.associations['PlatformModel']. associated_resources), 0) self.assertEqual( len(platform_data.associations['PlatformAgentInstance']. associated_resources), 0) self.assertEqual( len(platform_data.associations['InstrumentDevice'].resources), 1) platform_data = self.IMS.prepare_platform_device_support( platform_device_id) #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, platform_device_id) self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual( len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual( platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual( len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual( platform_data.associations['PlatformAgentInstance'].resources[0]. _id, platform_agent_instance_id) self.assertEqual( len(platform_data.associations['PlatformModel']. associated_resources), 1) self.assertEqual( platform_data.associations['PlatformModel']. associated_resources[0].s, platform_device_id) self.assertEqual( platform_data.associations['PlatformModel']. associated_resources[0].o, platform_model_id) self.assertEqual( len(platform_data.associations['PlatformAgentInstance']. associated_resources), 1) self.assertEqual( platform_data.associations['PlatformAgentInstance']. associated_resources[0].o, platform_agent_instance_id) self.assertEqual( platform_data.associations['PlatformAgentInstance']. associated_resources[0].s, platform_device_id) self.assertEqual( len(platform_data.associations['InstrumentDevice'].resources), 2) #self.assertEqual(len(platform_data.associations['InstrumentDevice'].associated_resources), 1) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].s, platform_device_id) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_device_id) self.assertEqual( platform_data.associations['PlatformModel'].assign_request. request_parameters['platform_device_id'], platform_device_id) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.RR2.pluck(sensor_device2_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_device(instrument_device2_id) self.IMS.force_delete_platform_agent_instance( platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_device(platform_device2_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_device(sensor_device2_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id)
def _start_driver(self, dvr_config): """ Instantiate the DataHandler based on the configuration Called from: InstrumentAgent._handler_uninitialized_initialize @param dvr_config The driver configuration, equivalent to self._dvr_config @retval None or error. """ # Get driver configuration and pid for test case. dvr_mod = get_safe(self._dvr_config, 'dvr_mod', None) dvr_cls = get_safe(self._dvr_config, 'dvr_cls', None) dh_cfg = get_safe(self._dvr_config, 'dh_cfg', {}) log.debug('_start_driver: dvr_mod.dvr_cls={0}.{1} dh_cfg={2}'.format(dvr_mod,dvr_cls,dh_cfg)) if not dvr_mod or not dvr_cls: raise InstDriverError('DataHandler module ({0}) and class ({1}) cannot be None'.format(dvr_mod, dvr_cls)) #CBM: Could also follow the "stream_config" pattern and simply copy the stream_id(s) from agent.stream_config into dh_cfg if not 'stream_id' in dh_cfg: raise InstDriverError('DataHandler config must contain dh_cfg.stream_id: {0}'.format(self._dvr_config)) # # TODO: Retrieve all resources needed by the DataHandler, they will be provided during configuration # ## Here to !!!! END from external_observatory_agent resreg_cli = ResourceRegistryServiceClient() # CBM: WTF - If not wrapped - exception DISAPPEARS!! Why? ext_ds_res = None try: ext_ds_res = resreg_cli.read(object_id=self.resource_id) log.info('Dataset Resource (id={0}): {1}'.format(self.resource_id,ext_ds_res)) except NotFound as ex: if not ext_ds_res and not get_safe(self._dvr_config, 'dh_cfg.TESTING'): raise InstDriverError('No resource associated with id = {0}'.format(self.resource_id)) dh_cfg['external_dataset_res'] = ext_ds_res dh_cfg['external_dataset_res_id'] = self.resource_id # ext_resources = {'dataset':ext_ds_res} # log.debug('Retrieved ExternalDataset: {0}'.format(ext_ds_res)) # # dsrc_res, dsrc_assn = resreg_cli.find_objects(subject=ext_dataset_id, predicate=PRED.hasSource, object_type=RT.DataSource) # dsrc_res = dsrc_res[0] # dsrc_id = dsrc_assn[0].o # ext_resources['datasource'] = dsrc_res # log.debug('Found associated DataSource: {0}'.format(dsrc_id)) # # edp_res, edp_assn = resreg_cli.find_objects(subject=dsrc_id, predicate=PRED.hasProvider, object_type=RT.ExternalDataProvider) # edp_res = edp_res[0] # edp_id = edp_assn[0].o # ext_resources['provider'] = edp_res # log.debug('Found associated ExternalDataProvider: {0}'.format(edp_id)) # # dsrc_mdl_res, dsrc_mdl_assn = resreg_cli.find_objects(subject=dsrc_id, predicate=PRED.hasModel, object_type=RT.DataSourceModel) # dsrc_mdl_res = dsrc_mdl_res[0] # dsrc_mdl_id = dsrc_mdl_assn[0].o # ext_resources['datasource_model'] = dsrc_mdl_res # log.debug('Found associated DataSourceModel: {0}'.format(dsrc_mdl_id)) # # dprod_res, dprod_assn = resreg_cli.find_objects(subject=ext_dataset_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct) # dprod_res = dprod_res[0] # dprod_id = dprod_assn[0].o # ext_resources['data_products'] = dprod_res # log.debug('Found associated DataProduct: {0}'.format(dprod_id)) # # stream_res, stream_assn = resreg_cli.find_objects(subject=dprod_id, predicate=PRED.hasStream, object_type=RT.Stream) # stream_res = stream_res[0] # stream_id = stream_assn[0].o # ext_resources['stream_res'] = stream_res # log.debug('Found associated Stream: {0}'.format(stream_id)) # # comms_config = {'dataset_id':self.resource_id,'resources':ext_resources} # ## !!!! END # # TODO: Add the bits the DataHandler needs to know about to the 'comms_config' portion of the _dvr_config # comms_config = {} # The 'comms_config' portion of dvr_config is passed to configure() # self._dvr_config['comms_config'] = comms_config # Instantiate the DataHandler based on the configuration try: module = __import__(dvr_mod, fromlist=[dvr_cls]) classobj = getattr(module, dvr_cls) log.debug('Load DataHandler: module={0} classojb={1}'.format(module,classobj)) self._dvr_client = classobj(self._rr_cli, self._stream_registrar, dh_cfg) self._dvr_client.set_event_callback(self.evt_recv) # Initialize the DataHandler self._dvr_client.cmd_dvr('initialize') except Exception as ex: self._dvr_client = None raise InstDriverError('Error instantiating DataHandler \'{0}.{1}\': {2}'.format(dvr_mod, dvr_cls, ex)) #TODO: Temporarily construct packet factories to utilize pathways provided by IA self._construct_packet_factories(dvr_mod) log.info('ExternalDatasetAgent \'{0}\' loaded DataHandler \'{1}.{2}\''.format(self._proc_name,dvr_mod,dvr_cls))
class IngestionManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.ingestion_management = IngestionManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.ingest_name = 'basic' self.exchange = 'testdata' @staticmethod def clean_subscriptions(): ingestion_management = IngestionManagementServiceClient() pubsub = PubsubManagementServiceClient() rr = ResourceRegistryServiceClient() ingestion_config_ids = ingestion_management.list_ingestion_configurations( id_only=True) for ic in ingestion_config_ids: subscription_ids, assocs = rr.find_objects( subject=ic, predicate=PRED.hasSubscription, id_only=True) for subscription_id, assoc in zip(subscription_ids, assocs): rr.delete_association(assoc) try: pubsub.deactivate_subscription(subscription_id) except: log.exception("Unable to decativate subscription: %s", subscription_id) pubsub.delete_subscription(subscription_id) def create_ingest_config(self): self.queue = IngestionQueue(name='test', type='testdata') # Create the ingestion config ingestion_config_id = self.ingestion_management.create_ingestion_configuration( name=self.ingest_name, exchange_point_id=self.exchange, queues=[self.queue]) return ingestion_config_id def test_ingestion_config_crud(self): ingestion_config_id = self.create_ingest_config() ingestion_config = self.ingestion_management.read_ingestion_configuration( ingestion_config_id) self.assertTrue(ingestion_config.name == self.ingest_name) self.assertTrue(ingestion_config.queues[0].name == 'test') self.assertTrue(ingestion_config.queues[0].type == 'testdata') ingestion_config.name = 'another' self.ingestion_management.update_ingestion_configuration( ingestion_config) # Create an association just to make sure that it will delete them sub = Subscription() sub_id, _ = self.resource_registry.create(sub) assoc_id, _ = self.resource_registry.create_association( subject=ingestion_config_id, predicate=PRED.hasSubscription, object=sub_id) self.ingestion_management.delete_ingestion_configuration( ingestion_config_id) with self.assertRaises(NotFound): self.resource_registry.read(assoc_id) def test_list_ingestion(self): # Create the ingest_config config_id = self.create_ingest_config() retval = self.ingestion_management.list_ingestion_configurations( id_only=True) # Nice thing about this is that it breaks if r2dm adds an ingest_config self.assertTrue(config_id in retval)
class TestResourceRegistry(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to bank service self.resource_registry_service = ResourceRegistryServiceClient() @unittest.skip('Represents a bug in storage/retrieval') def test_tuple_in_dict(self): # create a resource with a tuple saved in a dict transform_obj = IonObject(RT.Transform) transform_obj.configuration = {} transform_obj.configuration["tuple"] = ('STRING', ) transform_id, _ = self.resource_registry_service.create(transform_obj) # read the resource back returned_transform_obj = self.resource_registry_service.read( transform_id) self.assertEqual(transform_obj.configuration["tuple"], returned_transform_obj.configuration["tuple"]) def test_basics(self): # Sequence all the tests so that we can save numerous system start and stops self._do_test_crud() self._do_test_read_mult() self._do_test_lifecycle() self._do_test_attach() self._do_test_association() self._do_test_find_resources() self._do_test_find_objects_mult() def _do_test_crud(self): # Some quick registry tests # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", name="name", foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name", "foo": "bar"}) self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name"}, foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Instantiate an object obj = IonObject("UserInfo", name="name") # Can set attributes that aren't in the object's schema with self.assertRaises(AttributeError) as cm: setattr(obj, "foo", "bar") self.assertTrue( cm.exception.message == "'UserInfo' object has no attribute 'foo'") # Cam't call update with object that hasn't been persisted with self.assertRaises(BadRequest) as cm: self.resource_registry_service.update(obj) self.assertTrue( cm.exception.message.startswith( "Object does not have required '_id' or '_rev' attribute")) # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) # Cannot create object with _id and _rev fields pre-set with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create(read_obj) # Update object read_obj.name = "John Doe" self.resource_registry_service.update(read_obj) # Update should fail with revision mismatch with self.assertRaises(Conflict) as cm: self.resource_registry_service.update(read_obj) # Re-read and update object read_obj = self.resource_registry_service.read(obj_id) self.resource_registry_service.update(read_obj) # Delete object self.resource_registry_service.delete(obj_id) # Make sure read, update and delete report error with self.assertRaises(NotFound) as cm: self.resource_registry_service.read(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.update(read_obj) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) # Owner creation tests user = IonObject("ActorIdentity", name='user') uid, _ = self.resource_registry_service.create(user) inst = IonObject("InstrumentDevice", name='instrument') iid, _ = self.resource_registry_service.create( inst, headers={'ion-actor-id': str(uid)}) ids, _ = self.resource_registry_service.find_objects(iid, PRED.hasOwner, RT.ActorIdentity, id_only=True) self.assertEquals(len(ids), 1) assoc = self.resource_registry_service.read(ids[0]) self.resource_registry_service.delete(iid) with self.assertRaises(NotFound) as ex: assoc = self.resource_registry_service.read(iid) def _do_test_read_mult(self): test_resource1_id, _ = self.resource_registry_service.create( Resource(name='test1')) test_resource2_id, _ = self.resource_registry_service.create( Resource(name='test2')) res_list = [test_resource1_id, test_resource2_id] objects = self.resource_registry_service.read_mult(res_list) for o in objects: self.assertIsInstance(o, Resource) self.assertTrue(o._id in res_list) def _do_test_lifecycle(self): # Lifecycle tests att = IonObject("InstrumentDevice", name='mine', description='desc') rid, rev = self.resource_registry_service.create(att) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.name, att.name) self.assertEquals(att1.lcstate, LCS.DRAFT) self.assertEquals(att1.availability, AS.PRIVATE) new_state = self.resource_registry_service.execute_lifecycle_transition( rid, LCE.PLAN) self.assertEquals(new_state, lcstate(LCS.PLANNED, AS.PRIVATE)) att2 = self.resource_registry_service.read(rid) self.assertEquals(att2.lcstate, LCS.PLANNED) self.assertEquals(att2.availability, AS.PRIVATE) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.execute_lifecycle_transition( rid, LCE.UNANNOUNCE) self.assertTrue( "type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce" in cm.exception.message) new_state = self.resource_registry_service.execute_lifecycle_transition( rid, LCE.DEVELOP) self.assertEquals(new_state, lcstate(LCS.DEVELOPED, AS.PRIVATE)) with self.assertRaises(BadRequest): self.resource_registry_service.execute_lifecycle_transition( resource_id=rid, transition_event='NONE##') self.resource_registry_service.set_lifecycle_state( rid, lcstate(LCS.INTEGRATED, AS.PRIVATE)) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.lcstate, LCS.INTEGRATED) self.assertEquals(att1.availability, AS.PRIVATE) def _do_test_attach(self): binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82" # Owner creation tests instrument = IonObject("InstrumentDevice", name='instrument') iid, _ = self.resource_registry_service.create(instrument) att = Attachment(content=binary, attachment_type=AttachmentType.BLOB) aid1 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment( aid1, include_content=True) self.assertEquals(binary, att1.content) import base64 att = Attachment(content=base64.encodestring(binary), attachment_type=AttachmentType.ASCII) aid2 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment( aid2, include_content=True) self.assertEquals(binary, base64.decodestring(att1.content)) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid1, aid2]) att_ids = self.resource_registry_service.find_attachments( iid, id_only=True, descending=True) self.assertEquals(att_ids, [aid2, aid1]) att_ids = self.resource_registry_service.find_attachments( iid, id_only=True, descending=True, limit=1) self.assertEquals(att_ids, [aid2]) atts = self.resource_registry_service.find_attachments( iid, id_only=False, include_content=True, limit=1) self.assertEquals(atts[0].content, binary) self.resource_registry_service.delete_attachment(aid1) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid2]) self.resource_registry_service.delete_attachment(aid2) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, []) def _do_test_association(self): # Instantiate ActorIdentity object actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create( actor_identity_obj) read_actor_identity_obj = self.resource_registry_service.read( actor_identity_obj_id) # Instantiate UserInfo object user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create( user_info_obj) read_user_info_obj = self.resource_registry_service.read( user_info_obj_id) # Test create failures with self.assertRaises(AttributeError) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.bogus, user_info_obj_id) self.assertTrue(cm.exception.message == "bogus") # Predicate not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, None, user_info_obj_id) self.assertTrue( cm.exception.message == "Association must have all elements set") # Subject id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( None, PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Association must have all elements set") # Object id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, None) self.assertTrue( cm.exception.message == "Association must have all elements set") # Bad subject id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association( "bogus", PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # Bad object id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, "bogus") self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # _id missing from subject with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message.startswith("Subject id")) # _id missing from object with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj) self.assertTrue(cm.exception.message.startswith("Object id")) # Wrong subject type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( user_info_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal subject type UserInfo for predicate hasInfo") # Wrong object type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id) self.assertTrue( cm.exception.message == "Illegal object type ActorIdentity for predicate hasInfo") # Create two different association types between the same subject and predicate assoc_id1, assoc_rev1 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) # Read object, subject res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True) self.assertEquals(res_obj1, user_info_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id) self.assertEquals(res_obj2._id, actor_identity_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True) self.assertEquals(res_obj2, actor_identity_obj_id) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations( None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, id_only=False) ret3 = self.resource_registry_service.find_associations( predicate=PRED.hasInfo, id_only=False) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo, read_user_info_obj) ret2 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations( None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None, True) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, id_only=True) ret3 = self.resource_registry_service.find_associations( predicate=PRED.hasInfo, id_only=True) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(None, None, None) self.assertIn("Illegal parameters", cm.exception.message) # Find subjects (good cases) subj_ret1 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) subj_ret2 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_subjects( None, PRED.hasInfo, user_info_obj_id, True) subj_ret4 = self.resource_registry_service.find_subjects( None, None, read_user_info_obj, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_subjects( None, PRED.hasInfo, user_info_obj_id, False) subj_ret6 = self.resource_registry_service.find_subjects( None, None, read_user_info_obj, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find subjects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(None, None, None) self.assertTrue(cm.exception.message == "Must provide object") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.bogus, user_info_obj_id, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_subjects( RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.hasInfo, user_info_obj, True) self.assertTrue( cm.exception.message == "Object id not available in object") # Find objects (good cases) subj_ret1 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True) subj_ret2 = self.resource_registry_service.find_objects( read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, None, True) subj_ret4 = self.resource_registry_service.find_objects( actor_identity_obj_id, None, None, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, None, False) subj_ret6 = self.resource_registry_service.find_objects( read_actor_identity_obj, None, None, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find objects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(None, None, None) self.assertTrue(cm.exception.message == "Must provide subject") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects( actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue( cm.exception.message == "Object id not available in subject") # Get association (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(None, None, None) self.assertIn("Illegal parameters", cm.exception.message) assoc = self.resource_registry_service.get_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(assoc._id == assoc_id1) # Delete (bad cases) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete_association("bogus") self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # Delete other association self.resource_registry_service.delete_association(assoc_id1) # Delete resources self.resource_registry_service.delete(actor_identity_obj_id) self.resource_registry_service.delete(user_info_obj_id) def _do_test_find_resources(self): with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_resources( RT.UserInfo, LCS.DRAFT, "name", False) self.assertTrue( cm.exception.message == "find by name does not support lcstate") ret = self.resource_registry_service.find_resources( RT.UserInfo, None, "name", False) self.assertEquals(len(ret[0]), 0) # Instantiate an object obj = IonObject("InstrumentAgentInstance", name="name") # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) ret = self.resource_registry_service.find_resources( RT.InstrumentAgentInstance, None, "name", False) self.assertEquals(len(ret[0]), 1) self.assertEquals(ret[0][0]._id, read_obj._id) ret = self.resource_registry_service.find_resources( RT.InstrumentAgentInstance, LCS.DEPLOYED, None, False) self.assertEquals(len(ret[0]), 1) self.assertEquals(ret[0][0]._id, read_obj._id) def _do_test_find_objects_mult(self): dp = DataProcess() transform = Transform() pd = ProcessDefinition() dp_id, _ = self.resource_registry_service.create(dp) transform_id, _ = self.resource_registry_service.create(transform) pd_id, _ = self.resource_registry_service.create(pd) self.resource_registry_service.create_association( subject=dp_id, object=transform_id, predicate=PRED.hasTransform) self.resource_registry_service.create_association( subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition) results, _ = self.resource_registry_service.find_objects_mult( subjects=[dp_id], id_only=True) self.assertTrue(results == [transform_id]) results, _ = self.resource_registry_service.find_objects_mult( subjects=[dp_id, transform_id], id_only=True) results.sort() correct = [transform_id, pd_id] correct.sort() self.assertTrue(results == correct) @attr('EXT') def test_get_resource_extension(self): #Testing multiple instrument owners subject1 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" actor_identity_obj1 = IonObject(RT.ActorIdentity, {"name": subject1}) actor_id1, _ = self.resource_registry_service.create( actor_identity_obj1) user_info_obj1 = IonObject(RT.UserInfo, {"name": "Foo"}) user_info_id1, _ = self.resource_registry_service.create( user_info_obj1) self.resource_registry_service.create_association( actor_id1, PRED.hasInfo, user_info_id1) subject2 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256" actor_identity_obj2 = IonObject(RT.ActorIdentity, {"name": subject2}) actor_id2, _ = self.resource_registry_service.create( actor_identity_obj2) user_info_obj2 = IonObject(RT.UserInfo, {"name": "Foo2"}) user_info_id2, _ = self.resource_registry_service.create( user_info_obj2) self.resource_registry_service.create_association( actor_id2, PRED.hasInfo, user_info_id2) test_obj = IonObject(RT.InformationResource, {"name": "TestResource"}) test_obj_id, _ = self.resource_registry_service.create(test_obj) self.resource_registry_service.create_association( test_obj_id, PRED.hasOwner, actor_id1) self.resource_registry_service.create_association( test_obj_id, PRED.hasOwner, actor_id2) extended_resource = self.resource_registry_service.get_resource_extension( test_obj_id, OT.ExtendedInformationResource) self.assertEqual(test_obj_id, extended_resource._id) self.assertEqual(len(extended_resource.owners), 2) extended_resource_list = self.resource_registry_service.get_resource_extension( str([user_info_id1, user_info_id2]), OT.ExtendedInformationResource) self.assertEqual(len(extended_resource_list), 2) optional_args = {'user_id': user_info_id1} extended_resource = self.resource_registry_service.get_resource_extension( test_obj_id, OT.TestExtendedInformationResource, optional_args=optional_args) self.assertEqual(test_obj_id, extended_resource._id) self.assertEqual(len(extended_resource.owners), 2) self.assertEqual(extended_resource.user_id, user_info_id1) @attr('PREP') def test_prepare_resource_support(self): prepare_data = self.resource_registry_service.prepare_resource_support( resource_type=RT.StreamDefinition) self.assertEqual(prepare_data.create_request.service_name, "resource_registry") self.assertEqual(prepare_data.create_request.service_operation, "create") self.assertEqual(prepare_data.create_request.request_parameters, {"object": "$(object)"}) self.assertEqual(prepare_data.update_request.service_name, "resource_registry") self.assertEqual(prepare_data.update_request.service_operation, "update") self.assertEqual(prepare_data.update_request.request_parameters, {"object": "$(object)"}) res_id, _ = self.resource_registry_service.create( prepare_data.resource) prepare_data = self.resource_registry_service.prepare_resource_support( resource_type=RT.StreamDefinition, resource_id=res_id) prepare_data.resource.name = "test_stream_def" prepare_data.resource.stream_type = "test_type" stream_def_id, _ = self.resource_registry_service.update( prepare_data.resource) #def ion_object_encoder(obj): # return obj.__dict__ #print simplejson.dumps(prepare_data, default=ion_object_encoder, indent=2) stream_def = self.resource_registry_service.read(stream_def_id) self.assertEqual(stream_def.name, prepare_data.resource.name) self.assertEqual(stream_def.stream_type, prepare_data.resource.stream_type)
class TestResourceRegistryAttachments(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) print 'started services' def test_resource_registry_blob_sanity(self): resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo")) MY_CONTENT = "the quick brown fox etc etc etc" #save att_id = self.RR.create_attachment( resource_id, IonObject(RT.Attachment, name="test.txt", content=MY_CONTENT, content_type="text/plain", keywords=["test1", "test2"], attachment_type=AttachmentType.BLOB)) #load attachment = self.RR.read_attachment(att_id, include_content=True) self.assertEqual("test.txt", attachment.name) self.assertEqual("text/plain", attachment.content_type) self.assertIn("test1", attachment.keywords) self.assertIn("test2", attachment.keywords) #content has changed; it's base64-encoded from what we put in self.assertEqual(MY_CONTENT, attachment.content) obj = self.RR.read(resource_id) self.assertEqual(obj.name, "foo") obj.name = "TheDudeAbides" obj = self.RR.update(obj) obj = self.RR.read(resource_id) self.assertEqual(obj.name, "TheDudeAbides") att = self.RR.find_attachments(resource_id) self.assertNotEqual(att, None) actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.RR.create( actor_identity_obj) user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.RR.create(user_info_obj) assoc_id, assoc_rev = self.RR.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertNotEqual(assoc_id, None) find_assoc = self.RR.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(find_assoc[0]._id == assoc_id) subj = self.RR.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) res_obj1 = self.RR.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) self.RR.delete_association(assoc_id) self.RR.delete_attachment(att_id) self.RR.delete(resource_id)
class TestCTDTransformsIntegration(IonIntegrationTestCase): pdict_id = None def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataset_management = self.datasetclient def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_instrument_model(self): instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_agent_instance(self, instAgent_id, instDevice_id): port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) return instAgentInstance_id def _create_param_dicts(self): tdom, sdom = time_series_domain() self.sdom = sdom.dump() self.tdom = tdom.dump() self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) def _create_input_data_products( self, ctd_stream_def_id, instDevice_id, ): dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=self.tdom, spatial_domain=self.sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product( dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_parsed_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) #--------------------------------------------------------------------------- # Create CTD Raw as the second data product #--------------------------------------------------------------------------- if not self.pdict_id: self._create_param_dicts() raw_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_RAW', parameter_dictionary_id=self.pdict_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=self.tdom, spatial_domain=self.sdom) ctd_raw_data_product = self.dataproductclient.create_data_product( dp_obj, raw_stream_def_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_raw_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids return ctd_parsed_data_product def _create_data_process_definitions(self): #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L1 Conductivity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L1 Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L1 Temperature: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L2 Salinity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L2 Density: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) return self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id,\ self.ctd_L1_pressure_dprocdef_id,self.ctd_L1_temperature_dprocdef_id, \ self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id def _create_stream_definitions(self): if not self.pdict_id: self._create_param_dicts() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name='L0_Conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding='conductivity') outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name='L0_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding='pressure') outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name='L0_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding='temperature') return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id def _create_l0_output_data_products(self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id): out_data_prods = [] ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) out_data_prods.append(self.ctd_l0_conductivity_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_conductivity_output_dp_id) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) out_data_prods.append(self.ctd_l0_pressure_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_pressure_output_dp_id) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) out_data_prods.append(self.ctd_l0_temperature_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_temperature_output_dp_id) return out_data_prods def _create_l1_out_data_products(self): ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0]) self.loggerpids.append(pid) ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0]) self.loggerpids.append(pid) ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0]) self.loggerpids.append(pid) def _create_l2_out_data_products(self): #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- if not self.pdict_id: self._create_param_dicts() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition( name='L2_salinity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding='salinity') outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition( name='L2_Density', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding='density') ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0]) self.loggerpids.append(pid) ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0]) self.loggerpids.append(pid) @unittest.skip('This test errors on coi-nightly, may be OBE.') def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance( instAgent_id, instDevice_id) #------------------------------------------------------------------------------------- # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------- self._create_param_dicts() ctd_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=self.pdict_id) #------------------------------------------------------------------------------------- # Create two data products #------------------------------------------------------------------------------------- ctd_parsed_data_product = self._create_input_data_products( ctd_stream_def_id, instDevice_id) #------------------------------------------------------------------------------------- # Create data process definitions #------------------------------------------------------------------------------------- self._create_data_process_definitions() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------------------------------------------------------------- outgoing_stream_l0_conductivity_id, \ outgoing_stream_l0_pressure_id, \ outgoing_stream_l0_temperature_id = self._create_stream_definitions() self.out_prod_ids = self._create_l0_output_data_products( outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id) self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition( name='L1_conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding='conductivity') self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition( name='L1_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding='pressure') self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition( name='L1_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding='temperature') self._create_l1_out_data_products() self._create_l2_out_data_products() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------------------------------------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=self.ctd_L0_all_dprocdef_id, in_data_product_ids=[ctd_parsed_data_product], out_data_product_ids=self.out_prod_ids) self.dataprocessclient.activate_data_process( ctd_l0_all_data_process_id) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) extended_process = self.dataprocessclient.get_data_process_extension( ctd_l0_all_data_process_id) self.assertEquals(extended_process.computed.operational_state.status, ComputedValueAvailability.NOTAVAILABLE) self.assertEquals(data_process.message_controllable, True) #------------------------------------------------------------------------------------- # L1 Conductivity: Create the data process #------------------------------------------------------------------------------------- l1_conductivity_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_conductivity_dprocdef_id, [self.ctd_l0_conductivity_output_dp_id], [self.ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process( l1_conductivity_data_process_id) data_process = self.rrclient.read(l1_conductivity_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l1_conductivity_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L1 Pressure: Create the data process #------------------------------------------------------------------------------------- l1_pressure_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_pressure_dprocdef_id, [self.ctd_l0_pressure_output_dp_id], [self.ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process( l1_pressure_data_process_id) data_process = self.rrclient.read(l1_pressure_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l1_pressure_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L1 Temperature: Create the data process #------------------------------------------------------------------------------------- l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_temperature_dprocdef_id, [self.ctd_l0_temperature_output_dp_id], [self.ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process( l1_temperature_all_data_process_id) data_process = self.rrclient.read(l1_temperature_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l1_temperature_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Salinity: Create the data process #------------------------------------------------------------------------------------- l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process( l2_salinity_all_data_process_id) data_process = self.rrclient.read(l2_salinity_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l2_salinity_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Density: Create the data process #------------------------------------------------------------------------------------- l2_density_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_density_output_dp_id]) self.dataprocessclient.activate_data_process( l2_density_all_data_process_id) data_process = self.rrclient.read(l2_density_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l2_density_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) # Wait for instrument agent to spawn gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Streaming #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #todo ResourceAgentClient no longer has method set_param # # Make sure the sampling rate and transmission are sane. # params = { # SBE37Parameter.NAVG : 1, # SBE37Parameter.INTERVAL : 5, # SBE37Parameter.TXREALTIME : True # } # self._ia_client.set_param(params) #todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it? cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) #todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it? # log.debug("test_activateInstrumentStream: calling go_observatory") # cmd = AgentCommand(command='go_observatory') # reply = self._ia_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #------------------------------------------------------------------------------------------------- # Cleanup processes #------------------------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) #-------------------------------------------------------------------------------- # Cleanup data products #-------------------------------------------------------------------------------- dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct, id_only=True) for dp_id in dp_ids: self.dataproductclient.delete_data_product(dp_id)
class LastUpdateCacheTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.datastore_name = CACHE_DATASTORE_NAME self.container.start_rel_from_url('res/deploy/r2dm.yml') self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA) self.tms_cli = TransformManagementServiceClient() self.pubsub_cli = PubsubManagementServiceClient() self.pd_cli = ProcessDispatcherServiceClient() self.rr_cli = ResourceRegistryServiceClient() xs_dot_xp = CFG.core_xps.science_data try: self.XS, xp_base = xs_dot_xp.split('.') self.XP = '.'.join([bootstrap.get_sys_name(), xp_base]) except ValueError: raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp) def make_points(self,definition,stream_id='I am very special', N=100): from prototype.sci_data.constructor_apis import PointSupplementConstructor import numpy as np import random definition.stream_resource_id = stream_id total = N n = 10 # at most n records per granule i = 0 while i < total: r = random.randint(1,n) psc = PointSupplementConstructor(point_definition=definition, stream_id=stream_id) for x in xrange(r): i+=1 point_id = psc.add_point(time=i, location=(0,0,0)) psc.add_scalar_point_coverage(point_id=point_id, coverage_id='temperature', value=np.random.normal(loc=48.0,scale=4.0, size=1)[0]) psc.add_scalar_point_coverage(point_id=point_id, coverage_id='pressure', value=np.float32(1.0)) psc.add_scalar_point_coverage(point_id=point_id, coverage_id='conductivity', value=np.float32(2.0)) granule = psc.close_stream_granule() hdf_string = granule.identifiables[definition.data_stream_id].values granule.identifiables[definition.data_stream_id].values = hdf_string yield granule return def start_worker(self): proc_def = ProcessDefinition() proc_def.executable['module'] = 'ion.processes.data.last_update_cache' proc_def.executable['class'] = 'LastUpdateCache' proc_def_id = self.pd_cli.create_process_definition(process_definition=proc_def) subscription_id = self.pubsub_cli.create_subscription(query=ExchangeQuery(), exchange_name='ingestion_aggregate') config = { 'couch_storage' : { 'datastore_name' :self.datastore_name, 'datastore_profile' : 'SCIDATA' } } transform_id = self.tms_cli.create_transform( name='last_update_cache', description='LastUpdate that compiles an aggregate of metadata', in_subscription_id=subscription_id, process_definition_id=proc_def_id, configuration=config ) self.tms_cli.activate_transform(transform_id=transform_id) transform = self.rr_cli.read(transform_id) pid = transform.process_id handle = self.container.proc_manager.procs[pid] return handle @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_last_update_cache(self): handle = self.start_worker() queue = Queue() o_process = handle.process def new_process(msg): o_process(msg) queue.put(True) handle.process = new_process definition = SBE37_CDM_stream_definition() publisher = Publisher() stream_def_id = self.pubsub_cli.create_stream_definition(container=definition) stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id) time = float(0.0) for granule in self.make_points(definition=definition, stream_id=stream_id, N=10): publisher.publish(granule, to_name=(self.XP, stream_id+'.data')) # Determinism sucks try: queue.get(timeout=5) except Empty: self.assertTrue(False, 'Process never received the message.') doc = self.db.read(stream_id) ntime = doc.variables['time'].value self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.') time = ntime
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.dataset_management = self.datasetclient self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentModel id = %s', instModel_id) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD) try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") try: instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s', instDevice_id) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance") self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) log.info( 'TestDataProcessWithLookupTable: new Stream Definition id = %s', instDevice_id) log.info('Creating new CDM data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product( dp_obj, ctd_stream_def_id) log.info('new ctd_parsed_data_product_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.info('TestDataProcessWithLookupTable: Data product streams1 = %s', stream_ids) #------------------------------- # Create CTD Raw as the second data product #------------------------------- log.info( 'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition' ) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='ctd_raw', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) ctd_raw_data_product = self.dataproductclient.create_data_product( dp_obj, raw_stream_def_id) log.info('new ctd_raw_data_product_id = %s', ctd_raw_data_product) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.info('Data product streams2 = %s', stream_ids) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug( "TestDataProcessWithLookupTable: create data process definition ctd_L0_all" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new ctd_L0_all data process definition: %s" % ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment( ctd_L0_all_dprocdef_id, att) log.debug( "TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment)) processDefinitionAttachment_obj = self.rrclient.read( processDefinitionAttachment) log.debug( "TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj)) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity') outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure') outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature') log.debug( "TestDataProcessWithLookupTable: create output data product L0 conductivity" ) ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug( "TestDataProcessWithLookupTable: create output data product L0 pressure" ) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug( "TestDataProcessWithLookupTable: create output data product L0 temperature" ) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug( "TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) out_prods = [ ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id ] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=ctd_L0_all_dprocdef_id, in_data_product_ids=in_prods, out_data_product_ids=out_prods) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "TestDataProcessWithLookupTable: create L0 all data_process return" ) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, object_type=RT.Process, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment( ctd_l0_all_data_process_id, att) log.info( 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s', processAttachment)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node) def test_createDataProcess(self): #--------------------------------------------------------------------------- # Data Process Definition #--------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # Make assertion on the newly registered data process definition data_process_definition = self.rrclient.read(dprocdef_id) self.assertEquals(data_process_definition.name, 'ctd_L0_all') self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams') self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all') self.assertEquals(data_process_definition.class_name, 'ctd_L0_all') # Read the data process definition using data process management and make assertions dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id) self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all') self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all') #--------------------------------------------------------------------------- # Create an input instrument #--------------------------------------------------------------------------- instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) # Register the instrument so that the data producer and stream object are created data_producer_id = self.damsclient.register_instrument(instrument_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id ) # Assert that the link between the stream definition and the data process definition was done assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True) self.assertIsNotNone(assocs) #--------------------------------------------------------------------------- # Input Data Product #--------------------------------------------------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() input_dp_obj = IonObject( RT.DataProduct, name='InputDataProduct', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test') #Make assertions on the input data product created input_dp_obj = self.rrclient.read(input_dp_id) self.assertEquals(input_dp_obj.name, 'InputDataProduct') self.assertEquals(input_dp_obj.description, 'some new dp') self.damsclient.assign_data_product(instrument_id, input_dp_id) # Retrieve the stream via the DataProduct->Stream associations stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True) self.in_stream_id = stream_ids[0] #--------------------------------------------------------------------------- # Output Data Product #--------------------------------------------------------------------------- outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' ) outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' ) outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' ) self.output_products={} output_dp_obj = IonObject(RT.DataProduct, name='conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id) self.output_products['conductivity'] = output_dp_id_1 output_dp_obj = IonObject(RT.DataProduct, name='pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id) self.output_products['pressure'] = output_dp_id_2 output_dp_obj = IonObject(RT.DataProduct, name='temperature', description='transform output ', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id) self.output_products['temperature'] = output_dp_id_3 #--------------------------------------------------------------------------- # Create the data process #--------------------------------------------------------------------------- def _create_data_process(): dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products) return dproc_id dproc_id = _create_data_process() # Make assertions on the data process created data_process = self.dataprocessclient.read_data_process(dproc_id) # Assert that the data process has a process id attached self.assertIsNotNone(data_process.process_id) # Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute self.assertIsNotNone(data_process.input_subscription_id) output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True) self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3])) @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}}) def test_createDataProcessUsingSim(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg") instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) #------------------------------- # Create CTD Raw as the second data product #------------------------------- raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj.name = 'ctd_raw' ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # Create listener for data process events and verify that events are received. #------------------------------- # todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events #todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_id = data_process.process_id self.addCleanup(self.process_dispatcher.cancel_process, process_id) #------------------------------- # Wait until the process launched in the create_data_process() method is actually running, before proceeding further in this test #------------------------------- gate = ProcessStateGate(self.process_dispatcher.read_process, process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The data process (%s) did not spawn in 30 seconds" % process_id) #------------------------------- # Retrieve a list of all data process defintions in RR and validate that the DPD is listed #------------------------------- # todo: Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions # todo: data topics are being handled by pub sub at the level of streams self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) #todo: check that activate event is received L4-CI-SA-RQ-367 #todo... (it looks like no event is being published when the data process is activated... so below, we just check for now # todo... that the subscription is indeed activated) (Swarbhanu) # todo: monitor process to see if it is active (sa-rq-182) ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id) input_subscription_id = ctd_l0_all_data_process.input_subscription_id subs = self.rrclient.read(input_subscription_id) self.assertTrue(subs.activated) # todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the extended resources for data process definition and for data process #------------------------------- extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) self.assertEqual(1, len(extended_process_definition.data_processes)) log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition)) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEqual(1, len(extended_process.input_data_products)) log.debug("test_createDataProcess: extended_process %s", str(extended_process)) ################################ Test the removal of data processes ################################## #------------------------------------------------------------------- # Test the deleting of the data process #------------------------------------------------------------------- # Before deleting, get the input streams, output streams and the subscriptions so that they can be checked after deleting # dp_obj_1 = self.rrclient.read(ctd_l0_all_data_process_id) # input_subscription_id = dp_obj_1.input_subscription_id # out_prods, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) # in_prods, _ = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasInputProduct, id_only=True) # in_streams = [] # for in_prod in in_prods: # streams, _ = self.rrclient.find_objects(in_prod, PRED.hasStream, id_only=True) # in_streams.extend(streams) # out_streams = [] # for out_prod in out_prods: # streams, _ = self.rrclient.find_objects(out_prod, PRED.hasStream, id_only=True) # out_streams.extend(streams) # Deleting the data process self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id) # Check that the data process got removed. Check the lcs state. It should be retired dp_obj = self.rrclient.read(ctd_l0_all_data_process_id) self.assertEquals(dp_obj.lcstate, LCS.RETIRED) # Check for process defs still attached to the data process dpd_assn_ids = self.rrclient.find_associations(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcessDefinition, id_only=True) self.assertEquals(len(dpd_assn_ids), 0) # Check for output data product still attached to the data process out_products, assocs = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) self.assertEquals(len(out_products), 0) self.assertEquals(len(assocs), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check of the data process has been deactivated self.assertIsNone(dp_obj.input_subscription_id) # Read the original subscription id of the data process and check that it has been deactivated with self.assertRaises(NotFound): self.pubsubclient.read_subscription(input_subscription_id) #------------------------------------------------------------------- # Delete the data process definition #------------------------------------------------------------------- # before deleting, get the process definition being associated to in order to be able to check later if the latter gets deleted as it should proc_def_ids, proc_def_asocs = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasProcessDefinition) self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id) # check that the data process definition has been retired dp_proc_def = self.rrclient.read(ctd_L0_all_dprocdef_id) self.assertEquals(dp_proc_def.lcstate, LCS.RETIRED) # Check for old associations of this data process definition proc_defs, proc_def_asocs = self.rrclient.find_objects(ctd_L0_all_dprocdef_id, PRED.hasProcessDefinition) self.assertEquals(len(proc_defs), 0) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject= ctd_L0_all_dprocdef_id, id_only=True) self.assertEquals(len(obj_assns), 0) ################################ Test the removal of data processes ################################## # Try force delete... This should simply delete the associations and the data process object # from the resource registry #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id) #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process definition #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ datastore_name = CACHE_DATASTORE_NAME self.db = self.container.datastore_manager.get_datastore(datastore_name) self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Generic time-series data domain creation tdom, sdom = time_series_domain() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) # self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) self.get_datastore(dataset_ids[0]) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug("The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'") data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name,'DP1') self.assertEquals(data_product_object.description,'some new dp') log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id)
class TestExchangeManagementServiceInt(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2coi.yml') self.ems = ExchangeManagementServiceClient() self.rr = ResourceRegistryServiceClient() orglist, _ = self.rr.find_resources(RT.Org) if not len(orglist) == 1: raise StandardError("Unexpected number of orgs found") self.org_id = orglist[0]._id # we test actual exchange interaction in pyon, so it's fine to mock the broker interaction here self.container.ex_manager = Mock(exchange.ExchangeManager) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Test reaches into container, doesn\'t work with CEI') def test_xs_create_delete(self): exchange_space = ExchangeSpace(name="bobo") esid = self.ems.create_exchange_space(exchange_space, self.org_id) # should be able to pull from RR an exchange space es2 = self.rr.read(esid) self.assertEquals(exchange_space.name, es2.name) # should have an exchange declared on the broker self.container.ex_manager.create_xs.assert_called_once_with('bobo', use_ems=False) # should have an assoc to an org orglist, _ = self.rr.find_subjects(RT.Org, PRED.hasExchangeSpace, esid, id_only=True) self.assertEquals(len(orglist), 1) self.assertEquals(orglist[0], self.org_id) self.ems.delete_exchange_space(esid) # should no longer have that id in the RR self.assertRaises(NotFound, self.rr.read, esid) # should no longer have an assoc to an org orglist2, _ = self.rr.find_subjects(RT.Org, PRED.hasExchangeSpace, esid, id_only=True) self.assertEquals(len(orglist2), 0) # should no longer have that exchange declared self.assertEquals(self.container.ex_manager.delete_xs.call_count, 1) self.assertIn('bobo', self.container.ex_manager.delete_xs.call_args[0][0].exchange) # prefixed with sysname def test_xs_delete_without_create(self): self.assertRaises(NotFound, self.ems.delete_exchange_space, '123') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Test reaches into container, doesn\'t work with CEI') def test_xp_create_delete(self): # xp needs an xs first exchange_space = ExchangeSpace(name="doink") esid = self.ems.create_exchange_space(exchange_space, self.org_id) exchange_point = ExchangePoint(name="hammer") epid = self.ems.create_exchange_point(exchange_point, esid) # should be in RR ep2 = self.rr.read(epid) self.assertEquals(exchange_point.name, ep2.name) # should be associated to the XS as well xslist, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangePoint, epid, id_only=True) self.assertEquals(len(xslist), 1) self.assertEquals(xslist[0], esid) # should exist on broker (both xp and xs) self.container.ex_manager.create_xs.assert_called_once_with('doink', use_ems=False) self.assertIn('hammer', self.container.ex_manager.create_xp.call_args[0]) self.ems.delete_exchange_point(epid) self.ems.delete_exchange_space(esid) # should no longer be in RR self.assertRaises(NotFound, self.rr.read, epid) # should no longer be associated xslist2, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangePoint, epid, id_only=True) self.assertEquals(len(xslist2), 0) # should no longer exist on broker (both xp and xs) def test_xp_create_then_delete_xs(self): # xp needs an xs first exchange_space = ExchangeSpace(name="doink") esid = self.ems.create_exchange_space(exchange_space, self.org_id) exchange_point = ExchangePoint(name="hammer") epid = self.ems.create_exchange_point(exchange_point, esid) # delete xs self.ems.delete_exchange_space(esid) # should no longer have an association xslist2, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangePoint, epid, id_only=True) self.assertEquals(len(xslist2), 0) # TEST ONLY: have to clean up the xp or we leave junk on the broker # we have to do it manually because the xs is gone #self.ems.delete_exchange_point(epid) xs = exchange.ExchangeSpace(self.container.ex_manager, exchange_space.name) xp = exchange.ExchangePoint(self.container.ex_manager, exchange_point.name, xs, 'ttree') self.container.ex_manager.delete_xp(xp, use_ems=False) def test_xs_create_update(self): raise unittest.SkipTest("Test not implemented yet") def test_xn_declare_and_undeclare(self): # xn needs an xs first exchange_space = ExchangeSpace(name="bozo") esid = self.ems.create_exchange_space(exchange_space, self.org_id) exchange_name = ExchangeName(name='shoes', xn_type="XN_PROCESS") enid = self.ems.declare_exchange_name(exchange_name, esid) # should be in RR en2 = self.rr.read(enid) self.assertEquals(exchange_name.name, en2.name) # should have an assoc from XN to XS xnlist, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangeName, enid, id_only=True) self.assertEquals(len(xnlist), 1) self.assertEquals(xnlist[0], esid) # container API got called, will have declared a queue self.ems.undeclare_exchange_name(enid) # canonical name = xn id in current impl def test_xn_declare_no_xs(self): exchange_name = ExchangeName(name="shoez", xn_type='XN_PROCESS') self.assertRaises(NotFound, self.ems.declare_exchange_name, exchange_name, '11') def test_xn_undeclare_without_declare(self): self.assertRaises(NotFound, self.ems.undeclare_exchange_name, 'some_non_id') def test_xn_declare_then_delete_xs(self): # xn needs an xs first exchange_space = ExchangeSpace(name="bozo") esid = self.ems.create_exchange_space(exchange_space, self.org_id) exchange_name = ExchangeName(name='shnoz', xn_type="XN_PROCESS") enid = self.ems.declare_exchange_name(exchange_name, esid) # delete the XS self.ems.delete_exchange_space(esid) # no longer should have assoc from XS to XN xnlist, _ = self.rr.find_subjects(RT.ExchangeSpace, PRED.hasExchangeName, enid, id_only=True) self.assertEquals(len(xnlist), 0)
class DataRetrieverServiceIntTest(IonIntegrationTestCase): def setUp(self): super(DataRetrieverServiceIntTest,self).setUp() self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.couch = self.container.datastore_manager.get_datastore('test_data_retriever', profile=DataStore.DS_PROFILE.EXAMPLES) self.datastore_name = 'test_data_retriever' self.dr_cli = DataRetrieverServiceClient(node=self.container.node) self.dsm_cli = DatasetManagementServiceClient(node=self.container.node) self.rr_cli = ResourceRegistryServiceClient(node=self.container.node) self.ps_cli = PubsubManagementServiceClient(node=self.container.node) def tearDown(self): super(DataRetrieverServiceIntTest,self).tearDown() @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_define_replay(self): dataset_id = self.dsm_cli.create_dataset( stream_id='12345', datastore_name=self.datastore_name, view_name='posts/posts_join_comments', name='test define replay' ) replay_id, stream_id = self.dr_cli.define_replay(dataset_id=dataset_id) replay = self.rr_cli.read(replay_id) # Assert that the process was created self.assertTrue(self.container.proc_manager.procs[replay.process_id]) self.dr_cli.cancel_replay(replay_id) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_cancel_replay(self): dataset_id = self.dsm_cli.create_dataset( stream_id='12345', datastore_name=self.datastore_name, view_name='posts/posts_join_comments', name='test define replay' ) replay_id, stream_id = self.dr_cli.define_replay(dataset_id=dataset_id) replay = self.rr_cli.read(replay_id) # Assert that the process was created self.assertTrue(self.container.proc_manager.procs[replay.process_id]) self.dr_cli.cancel_replay(replay_id) # assert that the process is no more self.assertFalse(replay.process_id in self.container.proc_manager.procs) # assert that the resource no longer exists with self.assertRaises(NotFound): self.rr_cli.read(replay_id) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_start_replay(self): post = BlogPost(title='test blog post', post_id='12345', author=BlogAuthor(name='Jon Doe'), content='this is a blog post', updated=time.strftime("%Y-%m-%dT%H:%M%S-05")) dataset_id = self.dsm_cli.create_dataset( stream_id='12345', datastore_name=self.datastore_name, view_name='posts/posts_join_comments', name='blog posts test' ) self.couch.create(post) replay_id, stream_id = self.dr_cli.define_replay(dataset_id) replay = self.rr_cli.read(replay_id) # assert that the process was created self.assertTrue(self.container.proc_manager.procs[replay.process_id]) # pattern from Tim G ar = gevent.event.AsyncResult() def consume(message, headers): ar.set(message) stream_subscriber = StreamSubscriberRegistrar(process=self.container, node=self.container.node) subscriber = stream_subscriber.create_subscriber(exchange_name='test_queue', callback=consume) subscriber.start() query = StreamQuery(stream_ids=[stream_id]) subscription_id = self.ps_cli.create_subscription(query=query,exchange_name='test_queue') self.ps_cli.activate_subscription(subscription_id) self.dr_cli.start_replay(replay_id) self.assertEqual(ar.get(timeout=10).post_id,post.post_id) subscriber.stop() @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_chop_chop(self): # Override couch self.couch = self.container.datastore_manager.get_datastore( ds_name='chopping_block', profile=DataStore.DS_PROFILE.SCIDATA ) self.datastore_name = 'chopping_block' granule = ctd_stream_packet( stream_id='this_is_only_a_test', time='12345', #Same combo on my luggage create_hdf=False ) self.couch.create(granule) log.debug("Granule: %s", granule) dataset_id = self.dsm_cli.create_dataset( stream_id='this_is_only_a_test', datastore_name=self.datastore_name, view_name='datasets/dataset_by_id', name='sci_data_granule_chop' ) replay_id, stream_id = self.dr_cli.define_replay( dataset_id=dataset_id, delivery_format={'chop':True} ) replay = self.rr_cli.read(replay_id) self.assertTrue(self.container.proc_manager.procs[replay.process_id]) async_result = gevent.event.AsyncResult() def consume(message, headers): async_result.set(message) stream_subscriber = StreamSubscriberRegistrar(process=self.container, node=self.container.node) subscriber = stream_subscriber.create_subscriber(exchange_name = 'chopping_block', callback=consume) subscriber.start() query = StreamQuery(stream_ids=[stream_id]) subscription_id = self.ps_cli.create_subscription(query=query, exchange_name='chopping_block') self.ps_cli.activate_subscription(subscription_id=subscription_id) self.dr_cli.start_replay(replay_id) for fields in xrange(4): self.assertTrue(async_result.get(timeout=10)) subscriber.stop() self.dr_cli.cancel_replay(replay_id=replay_id)
class IndexManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url("res/deploy/r2dm.yml") self.ims_cli = IndexManagementServiceClient() self.rr_cli = ResourceRegistryServiceClient() self.index_name = "test_index" def test_create_elasticsearch_index(self): index_name = self.index_name ims_cli = self.ims_cli rr_cli = self.rr_cli options = SearchOptions() options.attribute_match = ["test_field"] index_id = ims_cli.create_index( name=index_name, content_type=IndexManagementService.ELASTICSEARCH_INDEX, options=options ) index_result = self.rr_cli.read(index_id) self.assertIsInstance(index_result, ElasticSearchIndex) self.assertTrue(index_result.name == index_name) # ====================================== # Clean up # ====================================== rr_cli.delete(index_id) def test_create_couchdb_index(self): index_name = self.index_name ims_cli = self.ims_cli rr_cli = self.rr_cli options = SearchOptions() options.attribute_match = ["name"] index_id = ims_cli.create_index( index_name, content_type=IndexManagementService.COUCHDB_INDEX, options=options, datastore_name="fake", view_name="fake/by_fake", ) index_result = self.rr_cli.read(index_id) self.assertIsInstance(index_result, CouchDBIndex) self.assertTrue(index_result.name == index_name) # ====================================== # Clean up # ====================================== rr_cli.delete(index_id) def test_read_index(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name index_res = Index(name=index_name) index_id, _ = rr_cli.create(index_res) index = ims_cli.read_index(index_id) self.assertIsInstance(index, Index) self.assertTrue(index.name == index_name) rr_cli.delete(index_id) def test_delete_index(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name index_res = Index(name=index_name) index_id, _ = rr_cli.create(index_res) ims_cli.delete_index(index_id) with self.assertRaises(NotFound): rr_cli.delete(index_id) def test_update_index(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name index_res = Index(name=index_name) index_id, _ = rr_cli.create(index_res) index = ims_cli.read_index(index_id) index.name = "another" ims_cli.update_index(index) index = rr_cli.read(index_id) self.assertTrue(index.name == "another") def test_find_indexes(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name # ====================================== # Index Pool # ====================================== indexes = [Index(name="first"), Index(name="second"), Index(name="third")] id_pool = list() for index in indexes: id_pool.append(rr_cli.create(index)[0]) index_id = ims_cli.find_indexes(index_name="second") index = ims_cli.read_index(index_id) self.assertTrue(index.name == "second") # ====================================== # Clean up # ====================================== for index_id in id_pool: rr_cli.delete(index_id) def test_list_indexes(self): ims_cli = self.ims_cli rr_cli = self.rr_cli index_name = self.index_name # ====================================== # Index Pool # ====================================== indexes = [Index(name="first"), Index(name="second"), Index(name="third")] id_pool = list() for index in indexes: id_pool.append(rr_cli.create(index)[0]) names = set(ims_cli.list_indexes().keys()) self.assertTrue(names == set(["first", "second", "third"])) # ====================================== # Clean up # ====================================== for index_id in id_pool: rr_cli.delete(index_id) def test_create_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli with self.assertRaises(BadRequest): ims_cli.create_collection("failing_collection") resources = [Resource(), Resource(), Resource()] resources = [rr_cli.create(i)[0] for i in resources] collection_id = ims_cli.create_collection("working_collection", resources) collection = rr_cli.read(collection_id) collection_resources = ims_cli.list_collection_resources(collection_id, id_only=True) self.assertTrue( set(collection_resources) == set(resources), "%s != %s" % (set(collection_resources), set(resources)) ) def test_read_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli collection = Collection(name="working_collection") collection_id, _ = rr_cli.create(collection) collection = ims_cli.read_collection(collection_id) self.assertTrue(collection.name == "working_collection") def test_update_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli collection = Collection(name="useful_collection") collection_id, _ = rr_cli.create(collection) collection = rr_cli.read(collection_id) collection.name = "nub" ims_cli.update_collection(collection) collection = rr_cli.read(collection_id) self.assertTrue(collection.name == "nub") def test_delete_collection(self): ims_cli = self.ims_cli rr_cli = self.rr_cli res = Resource() res_id, rev = rr_cli.create(res) collection_id = ims_cli.create_collection(name="test_collection", resources=[res_id]) ims_cli.delete_collection(collection_id) with self.assertRaises(NotFound): rr_cli.read(collection_id) def test_list_collection_resources(self): ims_cli = self.ims_cli rr_cli = self.rr_cli # ======================================== # Resource Pool # ======================================== resources = [ InformationResource(name="bean_counter"), InformationResource(name="lunar_rock"), InformationResource("aperature"), InformationResource("lemons"), ] resources = [rr_cli.create(i)[0] for i in resources] collection = Collection(name="park_bench") collection_id = ims_cli.create_collection(name="park_bench", resources=resources) retval = ims_cli.list_collection_resources(collection_id, id_only=True) retval.sort() resources.sort() self.assertTrue(retval == resources, "%s != %s" % (retval, resources)) def test_find_collection(self): res_id, _ = self.rr_cli.create(Resource(name="test_res")) collection_id = self.ims_cli.create_collection("test", [res_id]) retval = self.ims_cli.find_collection(collection_name="test") self.assertTrue(retval[0] == collection_id) retval = self.ims_cli.find_collection(resource_ids=[res_id]) self.assertTrue(retval[0] == collection_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dmpsclient = DataProductManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.psmsclient = PubsubManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.dsmsclient = DataProcessManagementServiceClient( node=self.container.node) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources( RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj)) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_prepare_deployment_support(self): deploy_sup = self.omsclient.prepare_deployment_support() self.assertTrue(deploy_sup) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].type_, "AssocDeploymentInstDevice") self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice'].type_, "AssocDeploymentPlatDevice") self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].type_, "AssocDeploymentInstSite") self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite'].type_, "AssocDeploymentPlatSite") self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite']. associated_resources, []) #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformDevice']. resources), 1) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformSite'].resources ), 1) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite']. associated_resources, []) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformDevice']. resources), 1) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformDevice']. associated_resources), 1) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformSite'].resources ), 1) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformSite']. associated_resources), 1) #delete the deployment self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self, make_assigns=False): # Create platform site, platform device, platform model bounds = GeospatialBounds(geospatial_latitude_limit_north=float(5), geospatial_latitude_limit_south=float(5), geospatial_longitude_limit_west=float(15), geospatial_longitude_limit_east=float(15), geospatial_vertical_min=float(0), geospatial_vertical_max=float(1000)) platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site', constraint_list=[bounds]) platform_site_id = self.omsclient.create_platform_site( platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device( platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model( platform_model__obj) # Create instrument site #------------------------------------------------------------------------------------- bounds = GeospatialBounds(geospatial_latitude_limit_north=float(45), geospatial_latitude_limit_south=float(40), geospatial_longitude_limit_west=float(-75), geospatial_longitude_limit_east=float(-70), geospatial_vertical_min=float(0), geospatial_vertical_max=float(500)) instrument_site_obj = IonObject( RT.InstrumentSite, name='InstrumentSite1', description='test instrument site', reference_designator='GA01SUMO-FI003-01-CTDMO0999', constraint_list=[bounds]) instrument_site_id = self.omsclient.create_instrument_site( instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) # Create an instrument device instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) pp_obj = IonObject(OT.PlatformPort, reference_designator='GA01SUMO-FI003-01-CTDMO0999', port_type=PortTypeEnum.PAYLOAD, ip_address='1') port_assignments = {instrument_device_id: pp_obj} #---------------------------------------------------------------------------------------------------- # Create an instrument model instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model( instrument_model_obj) # Create a deployment object #---------------------------------------------------------------------------------------------------- start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2020, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', context=IonObject( OT.CabledNodeDeploymentContext), port_assignments=port_assignments, constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) if make_assigns: self.imsclient.assign_platform_model_to_platform_device( platform_model_id, platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device( instrument_model_id, instrument_device_id) self.omsclient.assign_platform_model_to_platform_site( platform_model_id, platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site( instrument_model_id, instrument_site_id) self.omsclient.assign_site_to_deployment(platform_site_id, deployment_id) self.omsclient.assign_device_to_deployment(platform_device_id, deployment_id) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret def _create_subsequent_deployment(self, prior_dep_info): platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice2', description='test platform device') platform_device_id = self.imsclient.create_platform_device( platform_device_obj) instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice2', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.imsclient.assign_platform_model_to_platform_device( prior_dep_info.platform_model_id, platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device( prior_dep_info.instrument_model_id, instrument_device_id) start = str(int(time.mktime(datetime.datetime(2013, 6, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2020, 6, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment2', description='some new deployment', context=IonObject( OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment( prior_dep_info.platform_site_id, deployment_id) self.omsclient.assign_device_to_deployment( prior_dep_info.platform_device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) ret = DotDict(instrument_device_id=instrument_device_id, platform_device_id=platform_device_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment(make_assigns=True) before_activate_instrument_device_obj = self.rrclient.read( res.instrument_device_id) self.assertNotEquals(before_activate_instrument_device_obj.lcstate, LCS.DEPLOYED) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) def assertGeospatialBoundsEquals(a, b): self.assertEquals(a['geospatial_latitude_limit_north'], b['geospatial_latitude_limit_north']) self.assertEquals(a['geospatial_latitude_limit_south'], b['geospatial_latitude_limit_south']) self.assertEquals(a['geospatial_longitude_limit_west'], b['geospatial_longitude_limit_west']) self.assertEquals(a['geospatial_longitude_limit_east'], b['geospatial_longitude_limit_east']) def assertGeospatialBoundsNotEquals(a, b): self.assertNotEquals(a['geospatial_latitude_limit_north'], b['geospatial_latitude_limit_north']) self.assertNotEquals(a['geospatial_latitude_limit_south'], b['geospatial_latitude_limit_south']) self.assertNotEquals(a['geospatial_longitude_limit_west'], b['geospatial_longitude_limit_west']) self.assertNotEquals(a['geospatial_longitude_limit_east'], b['geospatial_longitude_limit_east']) after_activate_instrument_device_obj = self.rrclient.read( res.instrument_device_id) assertGeospatialBoundsNotEquals( before_activate_instrument_device_obj.geospatial_bounds, after_activate_instrument_device_obj.geospatial_bounds) deployment_obj = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj.lcstate, LCS.DEPLOYED) log.debug("deactivatin deployment, expecting success") self.omsclient.deactivate_deployment(res.deployment_id) after_deactivate_instrument_device_obj = self.rrclient.read( res.instrument_device_id) assertGeospatialBoundsNotEquals( after_activate_instrument_device_obj.geospatial_bounds, after_deactivate_instrument_device_obj.geospatial_bounds) deployment_obj = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj.lcstate, LCS.INTEGRATED) def test_activate_deployment_redeploy(self): dep_util = DeploymentUtil(self.container) res = self.base_activate_deployment(make_assigns=True) log.debug("activating first deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) deployment_obj1 = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj1.lcstate, LCS.DEPLOYED) next_dep_info = self._create_subsequent_deployment(res) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertNotEquals(deployment_obj2.lcstate, LCS.DEPLOYED) log.debug("activating subsequent deployment, expecting success") self.omsclient.activate_deployment(next_dep_info.deployment_id) deployment_obj1 = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj1.lcstate, LCS.INTEGRATED) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertEquals(deployment_obj2.lcstate, LCS.DEPLOYED) dep1_tc = dep_util.get_temporal_constraint(deployment_obj1) dep2_tc = dep_util.get_temporal_constraint(deployment_obj2) self.assertLessEqual(float(dep1_tc.end_datetime), float(dep2_tc.end_datetime)) log.debug("deactivating second deployment, expecting success") self.omsclient.deactivate_deployment(next_dep_info.deployment_id) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertEquals(deployment_obj2.lcstate, LCS.INTEGRATED) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.assign_site_to_deployment(res.platform_site_id, res.deployment_id) self.omsclient.assign_device_to_deployment(res.platform_device_id, res.deployment_id) log.debug( "activating deployment without site+device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug( "activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.omsclient.assign_device_to_deployment(res.instrument_device_id, res.deployment_id) log.debug( "activating deployment without instrument site, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest) #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.assign_site_to_deployment(res.instrument_site_id, res.deployment_id) log.debug("activating deployment without device, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment") def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message=""): with self.assertRaises(err_type) as cm: self.omsclient.activate_deployment(deployment_id) log.debug("assert_deploy_fail cm: %s", str(cm)) if fail_message: self.assertIn(fail_message, cm.exception.message) def test_3x3_matchups_remoteplatform(self): self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext)) def test_3x3_matchups_cabledinstrument(self): self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext)) def test_3x3_matchups_cablednode(self): self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext)) def base_3x3_matchups(self, deployment_context): """ This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1) """ deployment_context_type = type(deployment_context).__name__ instrument_model_id = [ self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6) ] platform_model_id = [ self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3) ] instrument_device_id = [ self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9) ] platform_device_id = [ self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4) ] instrument_site_id = [ self.RR2.create( any_old( RT.InstrumentSite, { "reference_designator": "GA01SUMO-FI003-0%s-CTDMO0999" % (i + 1), "planned_uplink_port": IonObject( OT.PlatformPort, reference_designator="GA01SUMO-FI003-0%s-CTDMO0999" % (i + 1)) })) for i in range(9) ] platform_site_id = [ self.RR2.create( any_old( RT.PlatformSite, { "reference_designator": "GA01SUMO-FI003-0%s-CTDMO0888" % (i + 1), "planned_uplink_port": IonObject( OT.PlatformPort, reference_designator="GA01SUMO-FI003-0%s-CTDMO0888" % (i + 1)) })) for i in range(4) ] def instrument_model_at(platform_idx, instrument_idx): m = platform_idx * 2 if instrument_idx > 0: m += 1 return m def platform_model_at(platform_idx): if platform_idx > 0: return 1 return 0 def instrument_at(platform_idx, instrument_idx): return platform_idx * 3 + instrument_idx # set up the structure for p in range(3): m = platform_model_at(p) self.RR2.assign_platform_model_to_platform_site_with_has_model( platform_model_id[m], platform_site_id[p]) self.RR2.assign_platform_model_to_platform_device_with_has_model( platform_model_id[m], platform_device_id[p]) self.RR2.assign_platform_device_to_platform_device_with_has_device( platform_device_id[p], platform_device_id[3]) self.RR2.assign_platform_site_to_platform_site_with_has_site( platform_site_id[p], platform_site_id[3]) for i in range(3): m = instrument_model_at(p, i) idx = instrument_at(p, i) self.RR2.assign_instrument_model_to_instrument_site_with_has_model( instrument_model_id[m], instrument_site_id[idx]) self.RR2.assign_instrument_model_to_instrument_device_with_has_model( instrument_model_id[m], instrument_device_id[idx]) self.RR2.assign_instrument_device_to_platform_device_with_has_device( instrument_device_id[idx], platform_device_id[p]) self.RR2.assign_instrument_site_to_platform_site_with_has_site( instrument_site_id[idx], platform_site_id[p]) # top level models self.RR2.assign_platform_model_to_platform_device_with_has_model( platform_model_id[2], platform_device_id[3]) self.RR2.assign_platform_model_to_platform_site_with_has_model( platform_model_id[2], platform_site_id[3]) # verify structure for p in range(3): parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device( platform_device_id[p]) self.assertEqual(platform_device_id[3], parent_id) parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site( platform_site_id[p]) self.assertEqual(platform_site_id[3], parent_id) for i in range(len(platform_site_id)): self.assertEqual( self.RR2. find_platform_model_of_platform_device_using_has_model( platform_device_id[i]), self.RR2.find_platform_model_of_platform_site_using_has_model( platform_site_id[i])) for i in range(len(instrument_site_id)): self.assertEqual( self.RR2. find_instrument_model_of_instrument_device_using_has_model( instrument_device_id[i]), self.RR2. find_instrument_model_of_instrument_site_using_has_model( instrument_site_id[i])) # OOIReferenceDesignator format: GA01SUMO-FI003-03-CTDMO0999 (site-platform_id-port-device_id) port_assignments = {} for p in range(3): ref_desig = "GA01SUMO-FI003-0%s-CTDMO0888" % (p + 1) pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type=PortTypeEnum.PAYLOAD, ip_address=str(p)) port_assignments[platform_device_id[p]] = pp_obj for i in range(3): ref_desig = "GA01SUMO-FI003-0%s-CTDMO0999" % ((p * 3) + i + 1) pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type=PortTypeEnum.PAYLOAD, ip_address=str(p)) idx = instrument_at(p, i) port_assignments[instrument_device_id[idx]] = pp_obj deployment_id = self.RR2.create( any_old( RT.Deployment, { "context": deployment_context, "port_assignments": port_assignments })) log.debug("assigning device/site to %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment( deployment_id, platform_device_id[3]) self.RR2.assign_deployment_to_platform_site_with_has_deployment( deployment_id, platform_site_id[3]) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_instrument_device_with_has_deployment( deployment_id, instrument_device_id[1]) self.RR2.assign_deployment_to_instrument_site_with_has_deployment( deployment_id, instrument_site_id[1]) elif OT.CabledNodeDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment( deployment_id, platform_device_id[1]) self.RR2.assign_deployment_to_platform_site_with_has_deployment( deployment_id, platform_site_id[1]) log.debug("activation of %s deployment", deployment_context_type) self.omsclient.activate_deployment(deployment_id) log.debug("validation of %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual( d, self.RR2. find_platform_device_id_of_platform_site_using_has_device( platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual( d, self.RR2. find_instrument_device_id_of_instrument_site_using_has_device( instrument_site_id[i])) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.assertEqual( instrument_device_id[1], self.RR2. find_instrument_device_id_of_instrument_site_using_has_device( instrument_site_id[1])) elif OT.CabledNodeDeploymentContext == deployment_context_type: expected_platforms = [1] # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual( i in expected_platforms, d in self.RR2. find_platform_device_ids_of_platform_site_using_has_device( platform_site_id[i]))
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dpsc_cli = DataProductManagementServiceClient() self.rrclient = ResourceRegistryServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.ingestclient = IngestionManagementServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() self.unsc = UserNotificationServiceClient() self.data_retriever = DataRetrieverServiceClient() self.identcli = IdentityManagementServiceClient() #------------------------------------------ # Create the environment #------------------------------------------ self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM') self.process_definitions = {} ingestion_worker_definition = ProcessDefinition(name='ingestion worker') ingestion_worker_definition.executable = { 'module':'ion.processes.data.ingestion.science_granule_ingestion_worker', 'class' :'ScienceGranuleIngestionWorker' } process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition) self.process_definitions['ingestion_worker'] = process_definition_id self.pids = [] self.exchange_points = [] self.exchange_names = [] #------------------------------------------------------------------------------------------------ # First launch the ingestors #------------------------------------------------------------------------------------------------ self.exchange_space = 'science_granule_ingestion' self.exchange_point = 'science_data' config = DotDict() config.process.datastore_name = 'datasets' config.process.queue_name = self.exchange_space self.exchange_names.append(self.exchange_space) self.exchange_points.append(self.exchange_point) pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config) log.debug("the ingestion worker process id: %s", pid) self.pids.append(pid) self.addCleanup(self.cleaning_up) def cleaning_up(self): for pid in self.pids: log.debug("number of pids to be terminated: %s", len(self.pids)) try: self.process_dispatcher.cancel_process(pid) log.debug("Terminated the process: %s", pid) except: log.debug("could not terminate the process id: %s" % pid) IngestionManagementIntTest.clean_subscriptions() for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore @attr('EXT') @attr('PREP') def test_create_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict') ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0 dp_obj.ooi_product_name = "PRODNAME" #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj, stream_definition_id=ctd_stream_def_id) # Assert that the data product has an associated stream at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertNotEquals(len(stream_ids), 0) # Assert that the data product has an associated stream def at this stage stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True) self.assertNotEquals(len(stream_ids), 0) self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Created data product %s', dp_obj) #------------------------------------------------------------------------------------------------ # test creating a new data product with a stream definition #------------------------------------------------------------------------------------------------ log.debug('Creating new data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='DP2', description='some new dp') dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id) self.dpsc_cli.activate_data_product_persistence(dp_id2) log.debug('new dp_id = %s' % dp_id2) #------------------------------------------------------------------------------------------------ #make sure data product is associated with stream def #------------------------------------------------------------------------------------------------ streamdefs = [] streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True) for s in streams: log.debug("Checking stream %s" % s) sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True) for sd in sdefs: log.debug("Checking streamdef %s" % sd) streamdefs.append(sd) self.assertIn(ctd_stream_def_id, streamdefs) group_names = self.dpsc_cli.get_data_product_group_list() self.assertIn("PRODNAME", group_names) #---------------------------------------------------------------------------------------- # Create users then notifications to this data product for each user #---------------------------------------------------------------------------------------- # user_1 user_1 = UserInfo() user_1.name = 'user_1' user_1.contact.email = '*****@*****.**' # user_2 user_2 = UserInfo() user_2.name = 'user_2' user_2.contact.email = '*****@*****.**' #user1 is a complete user self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject}) actor_id = self.identcli.create_actor_identity(actor_identity_obj) user_credentials_obj = IonObject("UserCredentials", {"name": self.subject}) self.identcli.register_user_credentials(actor_id, user_credentials_obj) user_id_1 = self.identcli.create_user_info(actor_id, user_1) user_id_2, _ = self.rrclient.create(user_2) delivery_config1a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) delivery_config1b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) notification_request_1 = NotificationRequest( name = "notification_1", origin=dp_id, origin_type="type_1", event_type=OT.ResourceLifecycleEvent, disabled_by_system = False, delivery_configurations=[delivery_config1a, delivery_config1b]) delivery_config2a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) delivery_config2b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH) notification_request_2 = NotificationRequest( name = "notification_2", origin=dp_id, origin_type="type_2", disabled_by_system = False, event_type=OT.DetectionEvent, delivery_configurations=[delivery_config2a, delivery_config2b]) notification_request_1_id = self.unsc.create_notification(notification=notification_request_1, user_id=user_id_1) notification_request_2_id = self.unsc.create_notification(notification=notification_request_2, user_id=user_id_2) self.unsc.delete_notification(notification_request_1_id) # test reading a non-existent data product log.debug('reading non-existent data product') with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product('some_fake_id') # update a data product (tests read also) log.debug('Updating data product') # first get the existing dp object dp_obj = self.dpsc_cli.read_data_product(dp_id) # now tweak the object dp_obj.description = 'the very first dp' dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0 dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0 dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0 # now write the dp back to the registry update_result = self.dpsc_cli.update_data_product(dp_obj) # now get the dp back to see if it was updated dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertEquals(dp_obj.description,'the very first dp') self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0) log.debug('Updated data product %s', dp_obj) #test extension extended_product = self.dpsc_cli.get_data_product_extension(dp_id) #validate that there is one active and one retired user notification for this data product self.assertEqual(1, len(extended_product.computed.active_user_subscriptions.value)) self.assertEqual(1, len(extended_product.computed.past_user_subscriptions.value)) self.assertEqual(dp_id, extended_product._id) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.product_download_size_estimated.status) self.assertEqual(0, extended_product.computed.product_download_size_estimated.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.computed.parameters.status) #log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value) def ion_object_encoder(obj): return obj.__dict__ #test prepare for create data_product_data = self.dpsc_cli.prepare_data_product_support() #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, "") self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0) #test prepare for update data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2) self.assertEqual(data_product_data._id, dp_id) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1) self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1) self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id) # now 'delete' the data product log.debug("deleting data product: %s" % dp_id) self.dpsc_cli.delete_data_product(dp_id) # Assert that there are no associated streams leftover after deleting the data product stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True) self.assertEquals(len(stream_ids), 0) self.assertEquals(len(assoc_ids), 0) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.dpsc_cli.read_data_product(dp_id) # Get the events corresponding to the data product ret = self.unsc.get_recent_events(resource_id=dp_id) events = ret.value for event in events: log.debug("event time: %s" % event.ts_created) self.assertTrue(len(events) > 0) def test_data_product_stream_def(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id) self.assertEquals(ctd_stream_def_id, stream_def_id) def test_derived_data_product(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id) dp = DataProduct(name='Instrument DP') dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id) self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id) self.dpsc_cli.activate_data_product_persistence(dp_id) self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Make the derived data product simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp']) tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED) tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id) self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id) # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) rdt['pressure'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True) tempwat_dataset_id = tempwat_dataset_ids[0] granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) self.assertEquals(set(rdt.fields), set(['time','temp'])) def test_activate_suspend_data_product(self): #------------------------------------------------------------------------------------------------ # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------------------ pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) log.debug("Created stream def id %s" % ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # test creating a new data product w/o a stream definition #------------------------------------------------------------------------------------------------ # Construct temporal and spatial Coordinate Reference System objects dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') log.debug("Created an IonObject for a data product: %s" % dp_obj) #------------------------------------------------------------------------------------------------ # Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary #------------------------------------------------------------------------------------------------ dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj, stream_definition_id=ctd_stream_def_id) #------------------------------------------------------------------------------------------------ # Subscribe to persist events #------------------------------------------------------------------------------------------------ queue = gevent.queue.Queue() def info_event_received(message, headers): queue.put(message) es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True) es.start() self.addCleanup(es.stop) #------------------------------------------------------------------------------------------------ # test activate and suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.activate_data_product_persistence(dp_id) dp_obj = self.dpsc_cli.read_data_product(dp_id) self.assertIsNotNone(dp_obj) dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True) if not dataset_ids: raise NotFound("Data Product %s dataset does not exist" % str(dp_id)) dataset_id = dataset_ids[0] # Check that the streams associated with the data product are persisted with stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True) for stream_id in stream_ids: self.assertTrue(self.ingestclient.is_persisted(stream_id)) stream_id = stream_ids[0] route = self.pubsubcli.read_stream_route(stream_id=stream_id) rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) publisher = StandaloneStreamPublisher(stream_id,route) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_ids[0]) self.assertIsInstance(replay_data, Granule) log.debug("The data retriever was able to replay the dataset that was attached to the data product " "we wanted to be persisted. Therefore the data product was indeed persisted with " "otherwise we could not have retrieved its dataset using the data retriever. Therefore " "this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'") data_product_object = self.rrclient.read(dp_id) self.assertEquals(data_product_object.name,'DP1') self.assertEquals(data_product_object.description,'some new dp') log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. " " Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the " "resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name, data_product_object.description)) #------------------------------------------------------------------------------------------------ # test suspend data product persistence #------------------------------------------------------------------------------------------------ self.dpsc_cli.suspend_data_product_persistence(dp_id) dataset_modified.clear() rdt['time'] = np.arange(20,40) publisher.publish(rdt.to_granule()) self.assertFalse(dataset_modified.wait(2)) self.dpsc_cli.activate_data_product_persistence(dp_id) dataset_modified.clear() publisher.publish(rdt.to_granule()) self.assertTrue(dataset_modified.wait(30)) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_almost_equal(rdt['time'], np.arange(40)) dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True) self.assertEquals(len(dataset_ids), 1) self.dpsc_cli.suspend_data_product_persistence(dp_id) self.dpsc_cli.force_delete_data_product(dp_id) # now try to get the deleted dp object with self.assertRaises(NotFound): dp_obj = self.rrclient.read(dp_id) info_event_counter = 0 runtime = 0 starttime = time.time() caught_events = [] #check that the four InfoStatusEvents were received while info_event_counter < 4 and runtime < 60 : a = queue.get(timeout=60) caught_events.append(a) info_event_counter += 1 runtime = time.time() - starttime self.assertEquals(info_event_counter, 4)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) print 'started services' # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.operational_state.status) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.power_status_roll_up.status) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.communications_status_roll_up.status) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_status_roll_up.status) self.assertEqual(StatusType.STATUS_OK, extended_instrument.computed.data_status_roll_up.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR resource_impl = ResourceImpl(c) resource_impl.pluck(instrument_agent_id) resource_impl.pluck(instrument_model_id) resource_impl.pluck(instrument_device_id) resource_impl.pluck(platform_agent_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_checkpoint_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", stream_configuration= {'raw': 'ctd_raw_param_dict' , 'parsed': 'ctd_parsed_param_dict' }) instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict') parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict') raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id, parameter_dictionary=spdict_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data self._get_datastore(self.parsed_dataset) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id, parameter_dictionary=rpdict_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.agent_state_checkpoint(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.agent_state_restore(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
class TestResourceRegistry(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url("res/deploy/r2coi.yml") # Now create client to bank service self.resource_registry_service = ResourceRegistryServiceClient(node=self.container.node) def test_crud(self): # Some quick registry tests # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", name="name", foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name", "foo": "bar"}) self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name"}, foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Instantiate an object obj = IonObject("UserInfo", name="name") # Can set attributes that aren't in the object's schema with self.assertRaises(AttributeError) as cm: setattr(obj, "foo", "bar") self.assertTrue(cm.exception.message == "'UserInfo' object has no attribute 'foo'") # Cam't call update with object that hasn't been persisted with self.assertRaises(BadRequest) as cm: self.resource_registry_service.update(obj) self.assertTrue(cm.exception.message.startswith("Object does not have required '_id' or '_rev' attribute")) # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) # Cannot create object with _id and _rev fields pre-set with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create(read_obj) self.assertTrue(cm.exception.message.startswith("Doc must not have '_id'")) # Update object read_obj.name = "John Doe" self.resource_registry_service.update(read_obj) # Update should fail with revision mismatch with self.assertRaises(Conflict) as cm: self.resource_registry_service.update(read_obj) self.assertTrue(cm.exception.message.startswith("Object not based on most current version")) # Re-read and update object read_obj = self.resource_registry_service.read(obj_id) self.resource_registry_service.update(read_obj) # Delete object self.resource_registry_service.delete(obj_id) # Make sure read, update and delete report error with self.assertRaises(NotFound) as cm: self.resource_registry_service.read(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.update(read_obj) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) # Owner creation tests user = IonObject("ActorIdentity", name="user") uid, _ = self.resource_registry_service.create(user) inst = IonObject("InstrumentDevice", name="instrument") iid, _ = self.resource_registry_service.create(inst, headers={"ion-actor-id": str(uid)}) ids, _ = self.resource_registry_service.find_objects(iid, PRED.hasOwner, RT.ActorIdentity, id_only=True) self.assertEquals(len(ids), 1) assoc = self.resource_registry_service.read(ids[0]) self.resource_registry_service.delete(iid) with self.assertRaises(NotFound) as ex: assoc = self.resource_registry_service.read(iid) def test_lifecycle(self): att = IonObject("InstrumentDevice", name="mine", description="desc") rid, rev = self.resource_registry_service.create(att) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.name, att.name) self.assertEquals(att1.lcstate, LCS.DRAFT_PRIVATE) new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.PLAN) self.assertEquals(new_state, LCS.PLANNED_PRIVATE) att2 = self.resource_registry_service.read(rid) self.assertEquals(att2.lcstate, LCS.PLANNED_PRIVATE) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.execute_lifecycle_transition(rid, LCE.UNANNOUNCE) self.assertTrue( "type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce" in cm.exception.message ) new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.DEVELOP) self.assertEquals(new_state, LCS.DEVELOPED_PRIVATE) self.assertRaises( iex.BadRequest, self.resource_registry_service.execute_lifecycle_transition, resource_id=rid, transition_event="NONE##", ) self.resource_registry_service.set_lifecycle_state(rid, LCS.INTEGRATED_PRIVATE) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.lcstate, LCS.INTEGRATED_PRIVATE) def test_association(self): # Instantiate ActorIdentity object actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create(actor_identity_obj) read_actor_identity_obj = self.resource_registry_service.read(actor_identity_obj_id) # Instantiate UserInfo object user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create(user_info_obj) read_user_info_obj = self.resource_registry_service.read(user_info_obj_id) # Test create failures with self.assertRaises(AttributeError) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.bogus, user_info_obj_id) self.assertTrue(cm.exception.message == "bogus") # Predicate not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Association must have all elements set") # Bad association type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, "bogustype" ) self.assertTrue(cm.exception.message == "Unsupported assoc_type: bogustype") # Subject id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(None, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Association must have all elements set") # Object id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, None) self.assertTrue(cm.exception.message == "Association must have all elements set") # Bad subject id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association("bogus", PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Object with id bogus does not exist.") # Bad object id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, "bogus") self.assertTrue(cm.exception.message == "Object with id bogus does not exist.") # _id missing from subject with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Subject id or rev not available") # _id missing from object with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj) self.assertTrue(cm.exception.message == "Object id or rev not available") # Wrong subject type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(user_info_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal subject type UserInfo for predicate hasInfo") # Wrong object type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id ) self.assertTrue(cm.exception.message == "Illegal object type ActorIdentity for predicate hasInfo") # Create two different association types between the same subject and predicate assoc_id1, assoc_rev1 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id ) # Read object, subject res_obj1 = self.resource_registry_service.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True ) self.assertEquals(res_obj1, user_info_obj_id) res_obj2 = self.resource_registry_service.read_subject(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id) self.assertEquals(res_obj2._id, actor_identity_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True ) self.assertEquals(res_obj2, actor_identity_obj_id) # Create a similar association to a specific revision # TODO: This is not a supported case so far assoc_id2, assoc_rev2 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, "H2R" ) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False ) ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=False) ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=False) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo, read_user_info_obj ) ret2 = self.resource_registry_service.find_associations(read_actor_identity_obj, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None, True ) ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=True) ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=True) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(None, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(actor_identity_obj_id, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(None, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(actor_identity_obj, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Object id not available in subject") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(actor_identity_obj_id, None, user_info_obj) self.assertTrue(cm.exception.message == "Object id not available in object") # Find subjects (good cases) subj_ret1 = self.resource_registry_service.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) subj_ret2 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True ) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, True) subj_ret4 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, False) subj_ret6 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find subjects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(None, None, None) self.assertTrue(cm.exception.message == "Must provide object") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.bogus, user_info_obj_id, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_subjects(RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj, True) self.assertTrue(cm.exception.message == "Object id not available in object") # Find objects (good cases) subj_ret1 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True) subj_ret2 = self.resource_registry_service.find_objects( read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True ) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, True) subj_ret4 = self.resource_registry_service.find_objects(actor_identity_obj_id, None, None, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, False) subj_ret6 = self.resource_registry_service.find_objects(read_actor_identity_obj, None, None, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find objects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(None, None, None) self.assertTrue(cm.exception.message == "Must provide subject") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True ) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue(cm.exception.message == "Object id not available in subject") # Get association (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(None, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(actor_identity_obj_id, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(None, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(actor_identity_obj, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Object id not available in subject") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(actor_identity_obj_id, None, user_info_obj) self.assertTrue(cm.exception.message == "Object id not available in object") # Delete one of the associations self.resource_registry_service.delete_association(assoc_id2) assoc = self.resource_registry_service.get_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(assoc._id == assoc_id1) # Delete (bad cases) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete_association("bogus") self.assertTrue(cm.exception.message == "Object with id bogus does not exist.") # Delete other association self.resource_registry_service.delete_association(assoc_id1) # Delete resources self.resource_registry_service.delete(actor_identity_obj_id) self.resource_registry_service.delete(user_info_obj_id) def test_find_resources(self): with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_resources(RT.UserInfo, LCS.DRAFT, "name", False) self.assertTrue(cm.exception.message == "find by name does not support lcstate") ret = self.resource_registry_service.find_resources(RT.UserInfo, None, "name", False) self.assertTrue(len(ret[0]) == 0) # Instantiate an object obj = IonObject("InstrumentDevice", name="name") # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) ret = self.resource_registry_service.find_resources(RT.InstrumentDevice, None, "name", False) self.assertTrue(len(ret[0]) == 1) self.assertTrue(ret[0][0]._id == read_obj._id) ret = self.resource_registry_service.find_resources(RT.InstrumentDevice, LCS.DRAFT, None, False) self.assertTrue(len(ret[0]) == 1) self.assertTrue(ret[0][0]._id == read_obj._id) # @attr('INT', group='coirr1') # class TestResourceRegistry1(IonIntegrationTestCase): # # def setUp(self): # # Start container # self._start_container() # self.container.start_rel_from_url('res/deploy/r2coi.yml') # # # Now create client to bank service # self.resource_registry_service = ResourceRegistryServiceClient(node=self.container.node) def test_attach(self): binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82" # Owner creation tests instrument = IonObject("InstrumentDevice", name="instrument") iid, _ = self.resource_registry_service.create(instrument) att = Attachment(content=binary, attachment_type=AttachmentType.BLOB) aid1 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid1) self.assertEquals(binary, att1.content) import base64 att = Attachment(content=base64.encodestring(binary), attachment_type=AttachmentType.ASCII) aid2 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid2) self.assertEquals(binary, base64.decodestring(att1.content)) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid1, aid2]) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True) self.assertEquals(att_ids, [aid2, aid1]) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True, limit=1) self.assertEquals(att_ids, [aid2]) atts = self.resource_registry_service.find_attachments(iid, id_only=False, limit=1) self.assertEquals(atts[0].content, att1.content) self.resource_registry_service.delete_attachment(aid1) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid2]) self.resource_registry_service.delete_attachment(aid2) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [])
class TestObservatoryManagementFullIntegration(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient() self.RR2 = EnhancedResourceRegistryClient(self.RR) self.OMS = ObservatoryManagementServiceClient() self.org_management_service = OrgManagementServiceClient() self.IMS = InstrumentManagementServiceClient() self.dpclient = DataProductManagementServiceClient() self.pubsubcli = PubsubManagementServiceClient() self.damsclient = DataAcquisitionManagementServiceClient() self.dataset_management = DatasetManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.data_product_management = DataProductManagementServiceClient() self._load_stage = 0 self._resources = {} def preload_ooi(self, stage=STAGE_LOAD_ASSETS): # Preloads OOI up to a given stage if self._load_stage >= stage: return if self._load_stage < STAGE_LOAD_ORGS: log.info("--------------------------------------------------------------------------------------------------------") log.info("Preloading stage: %s (OOIR2 Orgs, users, roles)", STAGE_LOAD_ORGS) # load_OOIR2_scenario self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="OOIR2", path="master", )) self._load_stage = STAGE_LOAD_ORGS if self._load_stage < STAGE_LOAD_PARAMS: log.info("--------------------------------------------------------------------------------------------------------") log.info("Preloading stage: %s (BASE params, streamdefs)", STAGE_LOAD_PARAMS) # load_parameter_scenarios self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories="ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) self._load_stage = STAGE_LOAD_PARAMS if self._load_stage < STAGE_LOAD_AGENTS: log.info("--------------------------------------------------------------------------------------------------------") log.info("Preloading stage: %s (OOIR2_I agents, model links)", STAGE_LOAD_AGENTS) # load_OOIR2_agents self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="OOIR2_I", path="master", )) self._load_stage = STAGE_LOAD_AGENTS if self._load_stage < STAGE_LOAD_ASSETS: log.info("--------------------------------------------------------------------------------------------------------") log.info("Preloading stage: %s (OOI assets linked to params, agents)", STAGE_LOAD_ASSETS) # load_ooi_assets self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", loadooi="True", path="master", assets="res/preload/r2_ioc/ooi_assets", bulk="True", debug="True", ooiuntil="9/1/2013", ooiparams="True", #excludecategories: DataProduct,DataProductLink,Deployment,Workflow,WorkflowDefinition )) self._load_stage = STAGE_LOAD_ASSETS # 'DataProduct,DataProductLink,WorkflowDefinition,ExternalDataProvider,ExternalDatasetModel,ExternalDataset,ExternalDatasetAgent,ExternalDatasetAgentInstance', @unittest.skip('Work in progress') def test_observatory(self): self._load_stage = 0 self._resources = {} passing = True self.assertTrue(True) # LOAD STEP 1 self.preload_ooi(stage=STAGE_LOAD_ORGS) passing &= self.orguserrole_assertions() # LOAD STEP 2 self.preload_ooi(stage=STAGE_LOAD_PARAMS) passing &= self.parameter_assertions() # LOAD STEP 3 self.preload_ooi(stage=STAGE_LOAD_AGENTS) passing &= self.agent_assertions() # LOAD STEP 4 self.preload_ooi(stage=STAGE_LOAD_ASSETS) # Check OOI preloaded resources to see if they match needs for this test and for correctness passing &= self.sites_assertions() passing &= self.device_assertions() passing &= self.deployment_assertions() # Extensive tests on select RSN nodes passing &= self.rsn_node_checks() # Extensive tests on select RSN instruments passing &= self.check_rsn_instrument() passing &= self.check_rsn_instrument_data_product() # Extensive tests on a glider #passing &= self.check_glider() # Extensive tests on a CG assembly #passing &= self.check_cg_assembly() # Add a new instrument agent # Add a new instrument agent instance # Check DataProducts # Check Provenance IonIntegrationTestCase.assertTrue(self, passing) # ------------------------------------------------------------------------- def orguserrole_assertions(self): passing = True passing &= self._check_marine_facility("MF_CGSN") passing &= self._check_marine_facility("MF_RSN") passing &= self._check_marine_facility("MF_EA") return passing def _check_marine_facility(self, preload_id): passing = True log.debug("Checking marine facility %s and associations", preload_id) mf_obj = self.retrieve_ooi_asset(preload_id) mf_id = mf_obj._id self._resources[preload_id] = mf_id passing &= self.assertEquals(mf_obj.lcstate, LCS.DEPLOYED) res_list, _ = self.RR.find_objects(subject=mf_id, predicate=PRED.hasMembership, id_only=True) passing &= self.assertTrue(len(res_list) >= 3) res_list, _ = self.RR.find_objects(subject=mf_id, predicate=PRED.hasRole, id_only=False) passing &= self.assertTrue(len(res_list) >= 5) passing &= self._check_role_assignments(res_list, "ORG_MANAGER") passing &= self._check_role_assignments(res_list, "OBSERVATORY_OPERATOR") passing &= self._check_role_assignments(res_list, "INSTRUMENT_OPERATOR") return passing def _check_role_assignments(self, role_list, role_name): passing = True role_obj = self._find_resource_in_list(role_list, "governance_name", role_name) if role_obj: res_list = self.RR.find_subjects(predicate=PRED.hasRole, object=role_obj._id, id_only=True) passing &= self.assertTrue(len(res_list) >= 1) return passing def parameter_assertions(self): passing = True pctx_list, _ = self.RR.find_resources_ext(restype=RT.ParameterContext) passing &= self.assertTrue(len(pctx_list) >= 10) pdict_list, _ = self.RR.find_resources_ext(restype=RT.ParameterDictionary) passing &= self.assertTrue(len(pdict_list) >= 10) sdef_list, _ = self.RR.find_resources_ext(restype=RT.StreamDefinition) passing &= self.assertTrue(len(sdef_list) >= 10) # Verify that a PDict has the appropriate QC parameters defined pdicts, _ = self.RR.find_resources_ext(restype=RT.ParameterDictionary, alt_id_ns='PRE', alt_id='DICT110') passing &= self.assertTrue(len(pdicts)==1) if not pdicts: return passing pdict = pdicts[0] # According to the latest SAF, density should NOT have trend parameters, _ = self.RR.find_objects(pdict, PRED.hasParameterContext) names = [i.name for i in parameters if i.name.startswith('density')] passing &= self.assertTrue('density_trndtst_qc' not in names) return passing def agent_assertions(self): passing = True # TODO: More tests? return passing def sites_assertions(self): passing = True observatory_list, _ = self.RR.find_resources_ext(restype=RT.Observatory) passing &= self.assertTrue(len(observatory_list) >= 40) for obs in observatory_list: passing &= self.assertEquals(obs.lcstate, LCS.DEPLOYED) platform_site_list, _ = self.RR.find_resources(RT.PlatformSite, id_only=False) log.debug('platform sites: %s', [ps.name for ps in platform_site_list]) passing &= self.assertTrue(len(platform_site_list) >= 30) return passing def device_assertions(self): passing = True platform_device_list, _ = self.RR.find_resources(RT.PlatformDevice, id_only=False) passing &= self.assertTrue(len(platform_device_list) >= 30) for pdev in platform_device_list: log.debug('platform device: %s', pdev.name) passing &= self.assertEquals(pdev.lcstate, LCS.PLANNED) platform_agent_list, _ = self.RR.find_resources(RT.PlatformAgent, id_only=False) passing &= self.assertTrue(len(platform_agent_list) >= 2) for pagent in platform_agent_list: log.debug('platform agent: %s', pagent.name) passing &= self.assertEquals(pagent.lcstate, LCS.DEPLOYED) instrument_agent_list, _ = self.RR.find_resources(RT.InstrumentAgent, id_only=False) passing &= self.assertTrue(len(instrument_agent_list) >= 3) for iagent in instrument_agent_list: log.debug('instrument agent: %s', iagent.name) passing &= self.assertEquals(iagent.lcstate, LCS.DEPLOYED) model_list, _ = self.RR.find_objects(subject=iagent._id, predicate=PRED.hasModel, id_only=True) passing &= self.assertTrue(len(model_list) >= 1, "IA %s" % iagent.name) return passing def deployment_assertions(self): passing = True deployment_list, _ = self.RR.find_resources(RT.Deployment, id_only=False) passing &= self.assertTrue(len(deployment_list) >= 30) for deploy in deployment_list: log.debug('deployment: %s', deploy.name) passing &= self.assertEquals(deploy.lcstate, LCS.DEPLOYED) return passing def rsn_node_checks(self): """ Current preload creates: - PlatformDevice in PLANNED - PlatformSite in DEPLOYED - Deployment in DEPLOYED - Deployment is NOT activated """ passing = True dp_obj = self.retrieve_ooi_asset("CE04OSHY-PN01C_DEP") passing &= self.assertEquals(dp_obj.lcstate, LCS.DEPLOYED) passing &= self.assertEquals(dp_obj.availability, AS.AVAILABLE) log.debug('test_observatory retrieve CE04OSHY-PN01C_DEP deployment: %s', dp_obj) # Check existing RSN node CE04OSHY-LV01C Deployment (PLANNED lcstate) CE04OSHY_LV01C_deployment = self.retrieve_ooi_asset('CE04OSHY-LV01C_DEP') passing &= self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, LCS.DEPLOYED) passing &= self.assertEquals(CE04OSHY_LV01C_deployment.availability, AS.AVAILABLE) #self.dump_deployment(CE04OSHY_LV01C_deployment._id) log.debug('test_observatory retrieve RSN node CE04OSHY-LV01C Deployment: %s', CE04OSHY_LV01C_deployment) CE04OSHY_LV01C_device = self.retrieve_ooi_asset('CE04OSHY-LV01C_PD') # Set CE04OSHY-LV01C device to DEVELOPED state passing &= self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.DEVELOP, verify=LCS.DEVELOPED) # Set CE04OSHY-LV01C device to INTEGRATED state passing &= self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.INTEGRATE, verify=LCS.INTEGRATED) # Set CE04OSHY-LV01C device to DEPLOYED state passing &= self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_device._id, new_lcs_state=LCE.DEPLOY, verify=LCS.DEPLOYED) # Set CE04OSHY-LV01C Deployment to DEPLOYED state # NOTE: Deployments are created in DEPLOYED state, currently #self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_deployment._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Activate Deployment for CE04OSHY-LV01C self.OMS.activate_deployment(CE04OSHY_LV01C_deployment._id) log.debug('--------- activate_deployment CE04OSHY_LV01C_deployment -------------- ') self.dump_deployment(CE04OSHY_LV01C_deployment._id) passing &= self.validate_deployment_activated(CE04OSHY_LV01C_deployment._id) # (optional) Start CE04OSHY-LV01C platform agent with simulator # NOTE: DataProduct is generated in DEPLOYED state # # Set DataProduct for CE04OSHY-LV01C platform to DEPLOYED state # output_data_product_ids, assns =self.RR.find_objects(subject=CE04OSHY_LV01C_device._id, predicate=PRED.hasOutputProduct, id_only=True) # if output_data_product_ids: # #self.assertEquals(len(child_devs), 3) # for output_data_product_id in output_data_product_ids: # log.debug('DataProduct for CE04OSHY-LV01C platform: %s', output_data_product_id) # self.transition_lcs_then_verify(resource_id=output_data_product_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Check events for CE04OSHY-LV01C platform # Check existing CE04OSBP-LJ01C Deployment (PLANNED lcstate) # dp_list, _ = self.RR.find_resources_ext(alt_id_ns="PRE", alt_id="CE04OSBP-LJ01C_DEP") # self.assertEquals(len(dp_list), 1) # CE04OSHY_LV01C_deployment = dp_list[0] # self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, 'PLANNED') # log.debug('test_observatory retrieve RSN node CE04OSBP-LJ01C Deployment: %s', CE04OSHY_LV01C_deployment) # Set CE04OSBP-LJ01C Deployment to DEPLOYED state # Update description and other attributes for CE04OSBP-LJ01C device resource # Create attachment (JPG image) for CE04OSBP-LJ01C device resource # Activate Deployment for CE04OSBP-LJ01C # (optional) Add/register CE04OSBP-LJ01C platform agent to parent agent # (optional) Start CE04OSBP-LJ01C platform agent return passing def check_rsn_instrument(self): """ Check existing RSN instrument CE04OSBP-LJ01C-06-CTDBPO108 Deployment (PLANNED lcstate) Current preload creates: - InstrumentDevice in PLANNED - InstrumentSite in DEPLOYED - Deployment in DEPLOYED - Deployment is activated """ passing = True CE04OSBP_LJ01C_06_CTDBPO108_deploy = self.retrieve_ooi_asset('CE04OSBP-LJ01C-06-CTDBPO108_DEP') self.dump_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) #passing &= self.assertEquals(CE04OSBP_LJ01C_06_CTDBPO108_deploy.lcstate, 'PLANNED') # Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEVELOPED state CE04OSBP_LJ01C_06_CTDBPO108_device = self.retrieve_ooi_asset('CE04OSBP-LJ01C-06-CTDBPO108_ID') passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEVELOP, verify='DEVELOPED') # Set CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED') # Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Set CE04OSBP-LJ01C-06-CTDBPO108 Deployment to DEPLOYED state #self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Activate Deployment for CE04OSBP-LJ01C-06-CTDBPO108 instrument log.debug('--------- activate_deployment CE04OSBP-LJ01C-06-CTDBPO108 deployment -------------- ') self.OMS.activate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) passing &= self.validate_deployment_activated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) # (optional) Add/register CE04OSBP-LJ01C-06-CTDBPO108 instrument agent to parent agent # (optional) Start CE04OSBP-LJ01C-06-CTDBPO108 instrument agent with simulator # Set all DataProducts for CE04OSBP-LJ01C-06-CTDBPO108 to DEPLOYED state # (optional) Create a substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device CE04OSBP_LJ01C_06_CTDBPO108_isite = self.retrieve_ooi_asset('CE04OSBP-LJ01C-06-CTDBPO108') ## create device here: retrieve CTD Mooring on Mooring Riser 001 - similiar? GP03FLMB_RI001_10_CTDMOG999_ID_idevice = self.retrieve_ooi_asset('GP03FLMB-RI001-10-CTDMOG999_ID') deploy_id_2 = self.create_basic_deployment(name='CE04OSBP-LJ01C-06-CTDBPO108_DEP2', description='substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device') self.IMS.deploy_instrument_device(instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_2) self.OMS.deploy_instrument_site(instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_2) self.dump_deployment(deploy_id_2) # (optional) Activate this second deployment - check first deployment is deactivated self.OMS.deactivate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) passing &= self.validate_deployment_deactivated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) # log.debug('Activate deployment deploy_id_2') # self.get_deployment_ids(deploy_id_2) # self.dump_deployment(deploy_id_2, "deploy_id_2") # self.OMS.activate_deployment(deploy_id_2) # passing &= self.validate_deployment_deactivated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id) # # # (optional) Set first CE04OSBP-LJ01C-06-CTDBPO108 Deployment to INTEGRATED state # passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED') # # # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state # passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED') # # # # (optional) Create a third Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a same device from first deployment # deploy_id_3 = self.create_basic_deployment(name='CE04OSBP-LJ01C-06-CTDBPO108_DEP3', description='substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with same device as first') # self.IMS.deploy_instrument_device(instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_3) # self.OMS.deploy_instrument_site(instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_3) # self.dump_deployment(deploy_id_3) # # # # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state # passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # # # (optional) Activate this third deployment - check second deployment is deactivated # log.debug('Activate deployment deploy_id_3') # self.dump_deployment(deploy_id_3) # self.OMS.activate_deployment(deploy_id_3) # #todo: check second deployment is deactivated return passing def check_data_product_reference(self, reference_designator, output=[]): passing = True data_product_ids, _ = self.RR.find_resources_ext(alt_id_ns='PRE', alt_id='%s_DPI1' % reference_designator, id_only=True) # Assuming DPI1 is parsed passing &= self.assertEquals(len(data_product_ids), 1) if not data_product_ids: return passing # Let's go ahead and activate it data_product_id = data_product_ids[0] self.dpclient.activate_data_product_persistence(data_product_id) self.addCleanup(self.dpclient.suspend_data_product_persistence, data_product_id) dataset_ids, _ = self.RR.find_objects(data_product_id, PRED.hasDataset, id_only=True) passing &= self.assertEquals(len(dataset_ids), 1) if not dataset_ids: return passing dataset_id = dataset_ids[0] stream_def_ids, _ = self.RR.find_objects(data_product_id, PRED.hasStreamDefinition, id_only=True) passing &= self.assertEquals(len(dataset_ids), 1) if not stream_def_ids: return passing stream_def_id = stream_def_ids[0] output.append((data_product_id, stream_def_id, dataset_id)) return passing def check_tempsf_instrument_data_product(self, reference_designator): passing = True info_list = [] passing &= self.check_data_product_reference(reference_designator, info_list) if not passing: return passing data_product_id, stream_def_id, dataset_id = info_list.pop() now = time.time() ntp_now = now + 2208988800 rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [ntp_now] rdt['temperature'] = [[ 25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873, 15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993, 10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098, 5.3456, 4.2994, 4.3009]] dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt) passing &= self.assertTrue(dataset_monitor.event.wait(20)) if not passing: return passing granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now]) passing &= self.assert_array_almost_equal(rdt['temperature'], [[ 25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873, 15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993, 10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098, 5.3456, 4.2994, 4.3009]]) return passing def check_trhph_instrument_data_products(self, reference_designator): passing = True info_list = [] passing &= self.check_data_product_reference(reference_designator, info_list) if not passing: return passing data_product_id, stream_def_id, dataset_id = info_list.pop() pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(stream_def_id) passing &= self.assertEquals(pdict.name, 'trhph_sample') rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) # calibration constants a = 1.98e-9 b = -2.45e-6 c = 9.28e-4 d = -0.0888 e = 0.731 V_s = 1.506 V_c = 0. T = 11.8 r1 = 0.906 r2 = 4.095 r3 = 4.095 ORP_V = 1.806 Cl = np.nan offset = 2008 gain = 4.0 # Normally this would be 50 per the DPS but the precision is %4.0f which truncates the values to the nearest 1... ORP = ((ORP_V * 1000.) - offset) / gain ntp_now = time.time() + 2208988800 rdt['cc_a'] = [a] rdt['cc_b'] = [b] rdt['cc_c'] = [c] rdt['cc_d'] = [d] rdt['cc_e'] = [e] rdt['ref_temp_volts'] = [V_s] rdt['resistivity_temp_volts'] = [V_c] rdt['eh_sensor'] = [ORP_V] rdt['resistivity_5'] = [r1] rdt['resistivity_x1'] = [r2] rdt['resistivity_x5'] = [r3] rdt['cc_offset'] = [offset] rdt['cc_gain'] = [gain] rdt['time'] = [ntp_now] passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'], [T], 2) passing &= self.assert_array_almost_equal(rdt['vent_fluid_chloride_conc'], [Cl], 4) passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP], 4) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt) passing &= self.assertTrue(dataset_monitor.event.wait(60)) if not passing: return passing granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'], [T], 2) passing &= self.assert_array_almost_equal(rdt['vent_fluid_chloride_conc'], [Cl], 4) passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP], 4) return passing def check_vel3d_instrument_data_products(self, reference_designator): passing = True info_list = [] passing &= self.check_data_product_reference(reference_designator, info_list) if not passing: return passing data_product_id, stream_def_id, dataset_id = info_list.pop() pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(stream_def_id) self.assertEquals(pdict.name, 'vel3d_b_sample') rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) lat = 14.6846 lon = -51.044 ts = np.array([3319563600, 3319567200, 3319570800, 3319574400, 3319578000, 3319581600, 3319585200, 3319588800, 3319592400, 3319596000], dtype=np.float) ve = np.array([ -3.2, 0.1, 0. , 2.3, -0.1, 5.6, 5.1, 5.8, 8.8, 10.3]) vn = np.array([ 18.2, 9.9, 12. , 6.6, 7.4, 3.4, -2.6, 0.2, -1.5, 4.1]) vu = np.array([-1.1, -0.6, -1.4, -2, -1.7, -2, 1.3, -1.6, -1.1, -4.5]) ve_expected = np.array([-0.085136, -0.028752, -0.036007, 0.002136, -0.023158, 0.043218, 0.056451, 0.054727, 0.088446, 0.085952]) vn_expected = np.array([ 0.164012, 0.094738, 0.114471, 0.06986, 0.07029, 0.049237, -0.009499, 0.019311, 0.012096, 0.070017]) vu_expected = np.array([-0.011, -0.006, -0.014, -0.02, -0.017, -0.02, 0.013, -0.016, -0.011, -0.045]) rdt['time'] = ts rdt['lat'] = [lat] * 10 rdt['lon'] = [lon] * 10 rdt['turbulent_velocity_east'] = ve rdt['turbulent_velocity_north'] = vn rdt['turbulent_velocity_up'] = vu passing &= self.assert_array_almost_equal(rdt['eastward_turbulent_velocity'], ve_expected) passing &= self.assert_array_almost_equal(rdt['northward_turbulent_velocity'], vn_expected) passing &= self.assert_array_almost_equal(rdt['upward_turbulent_velocity'], vu_expected) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt) passing &= self.assertTrue(dataset_monitor.event.wait(20)) if not passing: return passing granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) passing &= self.assert_array_almost_equal(rdt['eastward_turbulent_velocity'], ve_expected) passing &= self.assert_array_almost_equal(rdt['northward_turbulent_velocity'], vn_expected) passing &= self.assert_array_almost_equal(rdt['upward_turbulent_velocity'], vu_expected) return passing def check_presta_instrument_data_products(self, reference_designator): # Check the parsed data product make sure it's got everything it needs and can be published persisted etc. # Absolute Pressure (SFLPRES_L0) is what comes off the instrumnet, SFLPRES_L1 is a pfunc # Let's go ahead and publish some fake data!!! # According to https://alfresco.oceanobservatories.org/alfresco/d/d/workspace/SpacesStore/63e16865-9d9e-4b11-b0b3-d5658faa5080/1341-00230_Data_Product_Spec_SFLPRES_OOI.pdf # Appendix A. Example 1. # p_psia_tide = 14.8670 # the tide should be 10.2504 passing = True info_list = [] passing &= self.check_data_product_reference(reference_designator, info_list) if not passing: return passing data_product_id, stream_def_id, dataset_id = info_list.pop() now = time.time() ntp_now = now + 2208988800. rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [ntp_now] rdt['absolute_pressure'] = [14.8670] passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt) self.assertTrue(dataset_monitor.event.wait(20)) # Bumped to 20 to keep buildbot happy if not passing: return passing granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now]) passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4) passing &= self.assert_array_almost_equal(rdt['absolute_pressure'], [14.8670], 4) return passing def check_rsn_instrument_data_product(self): passing = True # for RS03AXBS-MJ03A-06-PRESTA301 (PREST-A) there are a few listed data products # Parsed, Engineering # SFLPRES-0 SFLPRES-1 # Check for the two data products and make sure they have the proper parameters # SFLPRES-0 should data_products, _ = self.RR.find_resources_ext(alt_id_ns='PRE', alt_id='RS03AXBS-MJ03A-06-PRESTA301_SFLPRES_L0_DPID', id_only=True) passing &=self.assertTrue(len(data_products)==1) if not data_products: return passing data_product_id = data_products[0] stream_defs, _ = self.RR.find_objects(data_product_id,PRED.hasStreamDefinition,id_only=False) passing &= self.assertTrue(len(stream_defs)==1) if not stream_defs: return passing # Assert that the stream definition has the correct reference designator stream_def = stream_defs[0] passing &= self.assertEquals(stream_def.stream_configuration['reference_designator'], 'RS03AXBS-MJ03A-06-PRESTA301') # Get the pdict and make sure that the parameters corresponding to the available fields # begin with the appropriate data product identifier pdict_ids, _ = self.RR.find_objects(stream_def, PRED.hasParameterDictionary, id_only=True) passing &= self.assertEquals(len(pdict_ids), 1) if not pdict_ids: return passing pdict_id = pdict_ids[0] pdict = DatasetManagementService.get_parameter_dictionary(pdict_id) available_params = [pdict.get_context(i) for i in pdict.keys() if i in stream_def.available_fields] for p in available_params: if p.name=='time': # Ignore the domain parameter continue passing &= self.assertTrue(p.ooi_short_name.startswith('SFLPRES')) passing &= self.check_presta_instrument_data_products('RS01SLBS-MJ01A-06-PRESTA101') passing &= self.check_vel3d_instrument_data_products( 'RS01SLBS-MJ01A-12-VEL3DB101') passing &= self.check_presta_instrument_data_products('RS03AXBS-MJ03A-06-PRESTA301') passing &= self.check_vel3d_instrument_data_products( 'RS03AXBS-MJ03A-12-VEL3DB301') passing &= self.check_tempsf_instrument_data_product( 'RS03ASHS-MJ03B-07-TMPSFA301') passing &= self.check_vel3d_instrument_data_products( 'RS03INT2-MJ03D-12-VEL3DB304') passing &= self.check_trhph_instrument_data_products( 'RS03INT1-MJ03C-10-TRHPHA301') self.data_product_management.activate_data_product_persistence(data_product_id) dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4) self.assert_array_almost_equal(rdt['absolute_pressure'], [14.8670], 4) self.data_product_management.suspend_data_product_persistence(data_product_id) # Should do nothing and not raise anything return passing def check_glider(self): ''' # Check that glider GP05MOAS-GL001 assembly is defined by OOI preload (3 instruments) ''' passing = True GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD') child_devs, assns =self.RR.find_objects(subject=GP05MOAS_GL001_device._id, predicate=PRED.hasDevice, id_only=True) passing &= self.assertEquals(len(child_devs), 3) # Set GP05MOAS-GL001 Deployment to DEPLOYED GP05MOAS_GL001_deploy = self.retrieve_ooi_asset('GP05MOAS-GL001_DEP') passing &= self.transition_lcs_then_verify(resource_id=GP05MOAS_GL001_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') # Activate Deployment for GP05MOAS-GL001 #self.OMS.activate_deployment(GP05MOAS_GL001_deploy._id) # Deactivate Deployment for GP05MOAS-GL001 #self.OMS.deactivate_deployment(GP05MOAS_GL001_deploy._id) # Create a new Deployment resource X without any assignment x_deploy_id = self.create_basic_deployment(name='X_Deployment', description='new Deployment resource X without any assignment') # Assign Deployment X to site GP05MOAS-GL001 GP05MOAS_GL001_psite = self.retrieve_ooi_asset('GP05MOAS-GL001') self.OMS.deploy_platform_site(GP05MOAS_GL001_psite._id, x_deploy_id) # Assign Deployment X to first device for GP05MOAS-GL001 GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD') self.IMS.deploy_platform_device(GP05MOAS_GL001_device._id, x_deploy_id) # Set GP05MOAS-GL001 Deployment to PLANNED state #self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.PLAN, verify='PLANNED') # ??? already in planned # Set second GP05MOAS-GL001 Deployment to DEPLOYED passing &= self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED') self.dump_deployment(x_deploy_id) # Activate second Deployment for GP05MOAS-GL001 #self.OMS.activate_deployment(x_deploy_id) # Deactivate second Deployment for GP05MOAS-GL001 #self.OMS.deactivate_deployment(x_deploy_id) return passing def check_cg_assembly(self): passing = True # Set several CE01ISSM-RI002-* instrument devices to DEVELOPED state # Assemble several CE01ISSM-RI002-* instruments to a CG CE01ISSM-RI002 component platform # Set several CE01ISSM-RI002-* instrument devices to INTEGRATED state # Assemble CE01ISSM-RI002 platform to CG CE01ISSM-LM001 station platform # Set CE01ISSM-RI002 component device to INTEGRATED state # Set CE01ISSM-LM001 station device to INTEGRATED state # Set CE01ISSM-LM001 station device to DEPLOYED state (children maybe too?) # Set CE01ISSM-LM001 Deployment to DEPLOYED # Activate CE01ISSM-LM001 platform assembly deployment # Dectivate CE01ISSM-LM001 platform assembly deployment # Set CE01ISSM-LM001 Deployment to INTEGRATED state # Set CE01ISSM-LM001 station device to INTEGRATED state # Set CE01ISSM-RI002 component device to INTEGRATED state # Set CE01ISSM-RI002 component device to INTEGRATED state # Disassemble CE01ISSM-RI002 platform from CG CE01ISSM-LM001 station platform # Disassemble all CE01ISSM-RI002-* instruments from a CG CE01ISSM-RI002 component platform # Retire instrument one for CE01ISSM-RI002-* # Retire device one for CE01ISSM-RI002 # Retire device one for CE01ISSM-LM001 return passing # ------------------------------------------------------------------------- def retrieve_ooi_asset(self, alt_id='', namespace='PRE'): dp_list, _ = self.RR.find_resources_ext(alt_id_ns=namespace, alt_id=alt_id) self.assertEquals(len(dp_list), 1) return dp_list[0] def transition_lcs_then_verify(self, resource_id, new_lcs_state, verify): ret = self.RR2.advance_lcs(resource_id, new_lcs_state) resource_obj = self.RR.read(resource_id) return self.assertEquals(resource_obj.lcstate, verify) def create_basic_deployment(self, name='', description=''): start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name=name, description=description, context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) return self.OMS.create_deployment(deployment_obj) def validate_deployment_activated(self, deployment_id=''): site_id, device_id = self.get_deployment_ids(deployment_id) assocs = self.RR.find_associations(subject=site_id, predicate=PRED.hasDevice, object=device_id) return self.assertEquals(len(assocs), 1) def validate_deployment_deactivated(self, deployment_id=''): site_id, device_id = self.get_deployment_ids(deployment_id) assocs = self.RR.find_associations(subject=site_id, predicate=PRED.hasDevice, object=device_id) return self.assertEquals(len(assocs), 0) def dump_deployment(self, deployment_id='', name=""): #site_id, device_id = self.get_deployment_ids(deployment_id) resource_list,_ = self.RR.find_subjects(predicate=PRED.hasDeployment, object=deployment_id, id_only=True) resource_list.append(deployment_id) resources = self.RR.read_mult(resource_list ) log.debug('--------- dump_deployment %s summary---------------', name) for resource in resources: log.debug('%s: %s (%s)', resource._get_type(), resource.name, resource._id) log.debug('--------- dump_deployment %s full dump ---------------', name) for resource in resources: log.debug('resource: %s ', resource) log.debug('--------- dump_deployment %s end ---------------', name) #assocs = self.container.resource_registry.find_assoctiations(anyside=deployment_id) # assocs = Container.instance.resource_registry.find_assoctiations(anyside=deployment_id) # log.debug('--------- dump_deployment ---------------') # for assoc in assocs: # log.debug('SUBJECT: %s PREDICATE: %s OBJET: %s', assoc.s, assoc.p, assoc.o) # log.debug('--------- dump_deployment end ---------------') def get_deployment_ids(self, deployment_id=''): devices = [] sites = [] idevice_list,_ = self.RR.find_subjects(RT.InstrumentDevice, PRED.hasDeployment, deployment_id, id_only=True) pdevice_list,_ = self.RR.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, id_only=True) devices = idevice_list + pdevice_list self.assertEquals(1, len(devices)) isite_list,_ = self.RR.find_subjects(RT.InstrumentSite, PRED.hasDeployment, deployment_id, id_only=True) psite_list,_ = self.RR.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, id_only=True) sites = isite_list + psite_list self.assertEquals(1, len(sites)) return sites[0], devices[0] def _find_resource_in_list(self, res_list, attr, attr_val, assert_found=True): for res in res_list: v = getattr(res, attr, None) if v == attr_val: return res if assert_found: self.assertTrue(False, "Attribute %s value %s not found in list" % (attr, attr_val)) return None # ------------------------------------------------------------------------- def _get_caller(self): s = inspect.stack() return "%s:%s" % (s[2][1], s[2][2]) @assertion_wrapper def assert_array_almost_equal(self, *args, **kwargs): np.testing.assert_array_almost_equal(*args, **kwargs) @assertion_wrapper def assertEquals(self, *args, **kwargs): IonIntegrationTestCase.assertEquals(self, *args, **kwargs) @assertion_wrapper def assertTrue(self, *args, **kwargs): IonIntegrationTestCase.assertTrue(self, *args, **kwargs)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient def test_createDataProcess(self): #--------------------------------------------------------------------------- # Data Process Definition #--------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # Make assertion on the newly registered data process definition data_process_definition = self.rrclient.read(dprocdef_id) self.assertEquals(data_process_definition.name, 'ctd_L0_all') self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams') self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all') self.assertEquals(data_process_definition.class_name, 'ctd_L0_all') # Read the data process definition using data process management and make assertions dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id) self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all') self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all') #--------------------------------------------------------------------------- # Create an input instrument #--------------------------------------------------------------------------- instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) # Register the instrument so that the data producer and stream object are created data_producer_id = self.damsclient.register_instrument(instrument_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id ) # Assert that the link between the stream definition and the data process definition was done assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True) self.assertIsNotNone(assocs) #--------------------------------------------------------------------------- # Input Data Product #--------------------------------------------------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() input_dp_obj = IonObject( RT.DataProduct, name='InputDataProduct', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test') #Make assertions on the input data product created input_dp_obj = self.rrclient.read(input_dp_id) self.assertEquals(input_dp_obj.name, 'InputDataProduct') self.assertEquals(input_dp_obj.description, 'some new dp') self.damsclient.assign_data_product(instrument_id, input_dp_id) # Retrieve the stream via the DataProduct->Stream associations stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True) self.in_stream_id = stream_ids[0] #--------------------------------------------------------------------------- # Output Data Product #--------------------------------------------------------------------------- outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' ) outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' ) outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' ) self.output_products={} output_dp_obj = IonObject(RT.DataProduct, name='conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id) self.output_products['conductivity'] = output_dp_id_1 output_dp_obj = IonObject(RT.DataProduct, name='pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id) self.output_products['pressure'] = output_dp_id_2 output_dp_obj = IonObject(RT.DataProduct, name='temperature', description='transform output ', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id) self.output_products['temperature'] = output_dp_id_3 #--------------------------------------------------------------------------- # Create the data process #--------------------------------------------------------------------------- def _create_data_process(): dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products) return dproc_id dproc_id = _create_data_process() # Make assertions on the data process created data_process = self.dataprocessclient.read_data_process(dproc_id) # Assert that the data process has a process id attached self.assertIsNotNone(data_process.process_id) # Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute self.assertIsNotNone(data_process.input_subscription_id) output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True) self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3])) @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}}) def test_createDataProcessUsingSim(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" ) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", svr_addr="localhost", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) #------------------------------- # Create CTD Raw as the second data product #------------------------------- raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj.name = 'ctd_raw' ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # Create listener for data process events and verify that events are received. #------------------------------- # todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events #todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) #------------------------------- # Retrieve a list of all data process defintions in RR and validate that the DPD is listed #------------------------------- # todo: add this validate for Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions # todo: This capability is not yet completed (Swarbhanu) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) #todo: check that activate event is received L4-CI-SA-RQ-367 #todo... (it looks like no event is being published when the data process is activated... so below, we just check for now # todo... that the subscription is indeed activated) (Swarbhanu) # todo: monitor process to see if it is active (sa-rq-182) ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id) input_subscription_id = ctd_l0_all_data_process.input_subscription_id subs = self.rrclient.read(input_subscription_id) self.assertTrue(subs.activated) # todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the extended resources for data process definition and for data process #------------------------------- extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) self.assertEqual(1, len(extended_process_definition.data_processes)) log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition)) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEqual(1, len(extended_process.input_data_products)) log.debug("test_createDataProcess: extended_process %s", str(extended_process)) #------------------------------- # Cleanup #------------------------------- self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id) self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id) self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id) self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) print 'started services' # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(StatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config(self): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) iconfig_builder = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_id = self.RR2.create(any_old(RT.Org)) inst_startup_config = {'startup': 'config'} required_config_keys = [ 'org_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'alarm_defs', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('stream_config', config) if None is inst_device_id: for key in ['alarm_defs', 'children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure() platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj) child_config = pconfig_builder.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure() platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id) child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) def sample_nested_platform_agent_instance_config(self): """ for informational purposes """ ret = {'org_name': 'Org_1', 'alarm_defs': {}, 'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)}, 'stream_config': {'parameter_dictionary': 'lots of stuff'}, 'agent': {'resource_id': '33e54106c4444444862da082098bc123'}, 'startup_config': {}, 'device_type': 'PlatformDevice', 'children': {'76a39596eeff4fd5b409c4cb93f0e581': {'org_name': 'Org_1', 'alarm_defs': {}, 'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)}, 'stream_config': {'parameter_dictionary': 'lots of stuff'}, 'agent': {'resource_id': '76a39596eeff4fd5b409c4cb93f0e581'}, 'startup_config': {}, 'device_type': 'PlatformDevice', 'children': {}}}} return ret