def _update_new_data_check_attachment(cls, res_id, new_content): """ Update the list of data files for the external dataset resource @param res_id the ID of the external dataset resource @param new_content list of new files found on data host @throws InstrumentException if external dataset resource can't be found @retval the attachment ID """ rr_cli = ResourceRegistryServiceClient() try: # Delete any attachments with the "NewDataCheck" keyword attachment_objs = rr_cli.find_attachments(resource_id=res_id, include_content=False, id_only=False) for attachment_obj in attachment_objs: kwds = set(attachment_obj.keywords) if 'NewDataCheck' in kwds: log.debug('Delete NewDataCheck attachment: {0}'.format(attachment_obj._id)) rr_cli.delete_attachment(attachment_obj._id) else: log.debug('Found attachment: {0}'.format(attachment_obj)) # Create the new attachment att = Attachment(name='new_data_check', attachment_type=AttachmentType.ASCII, keywords=['NewDataCheck', ], content_type='text/plain', content=msgpack.packb(new_content)) att_id = rr_cli.create_attachment(resource_id=res_id, attachment=att) except NotFound: raise InstrumentException('ExternalDatasetResource \'{0}\' not found'.format(res_id)) return att_id
def _update_new_data_check_attachment(cls, res_id, new_content): rr_cli = ResourceRegistryServiceClient() try: # Delete any attachments with the "NewDataCheck" keyword attachment_objs = rr_cli.find_attachments(resource_id=res_id, include_content=False, id_only=False) for attachment_obj in attachment_objs: kwds = set(attachment_obj.keywords) if 'NewDataCheck' in kwds: log.debug('Delete NewDataCheck attachment: {0}'.format(attachment_obj._id)) rr_cli.delete_attachment(attachment_obj._id) else: log.debug('Found attachment: {0}'.format(attachment_obj)) # Create the new attachment att = Attachment(name='new_data_check', attachment_type=AttachmentType.OBJECT, keywords=['NewDataCheck',], content_type='text/plain', content=new_content) att_id = rr_cli.create_attachment(resource_id=res_id, attachment=att) except NotFound: raise InstrumentException('ExternalDatasetResource \'{0}\' not found'.format(res_id)) return att_id
class TestResourceRegistryAttachments(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) print 'started services' def test_resource_registry_blob_sanity(self): resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo")) MY_CONTENT = "the quick brown fox etc etc etc" #save att_id = self.RR.create_attachment( resource_id, IonObject(RT.Attachment, name="test.txt", content=MY_CONTENT, content_type="text/plain", keywords=["test1", "test2"], attachment_type=AttachmentType.BLOB)) #load attachment = self.RR.read_attachment(att_id) self.assertEqual("test.txt", attachment.name) self.assertEqual("text/plain", attachment.content_type) self.assertIn("test1", attachment.keywords) self.assertIn("test2", attachment.keywords) #content has changed; it's base64-encoded from what we put in self.assertEqual(MY_CONTENT, attachment.content)
class TestResourceRegistryAttachments(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) print 'started services' def test_resource_registry_blob_sanity(self): resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo")) MY_CONTENT = "the quick brown fox etc etc etc" #save att_id = self.RR.create_attachment(resource_id, IonObject(RT.Attachment, name="test.txt", content=MY_CONTENT, content_type="text/plain", keywords=["test1", "test2"], attachment_type=AttachmentType.BLOB)) #load attachment = self.RR.read_attachment(att_id) self.assertEqual("test.txt", attachment.name) self.assertEqual("text/plain", attachment.content_type) self.assertIn("test1", attachment.keywords) self.assertIn("test2", attachment.keywords) #content has changed; it's base64-encoded from what we put in self.assertEqual(MY_CONTENT, attachment.content)
class TestResourceRegistryAttachments(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) print 'started services' def test_resource_registry_blob_sanity(self): resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo")) MY_CONTENT = "the quick brown fox etc etc etc" #save att_id = self.RR.create_attachment(resource_id, IonObject(RT.Attachment, name="test.txt", content=MY_CONTENT, content_type="text/plain", keywords=["test1", "test2"], attachment_type=AttachmentType.BLOB)) #load attachment = self.RR.read_attachment(att_id, include_content=True) self.assertEqual("test.txt", attachment.name) self.assertEqual("text/plain", attachment.content_type) self.assertIn("test1", attachment.keywords) self.assertIn("test2", attachment.keywords) #content has changed; it's base64-encoded from what we put in self.assertEqual(MY_CONTENT, attachment.content) obj = self.RR.read(resource_id) self.assertEqual(obj.name, "foo") obj.name = "TheDudeAbides" obj = self.RR.update(obj) obj = self.RR.read(resource_id) self.assertEqual(obj.name, "TheDudeAbides") att = self.RR.find_attachments(resource_id) self.assertNotEqual(att, None) actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.RR.create(actor_identity_obj) user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.RR.create(user_info_obj) assoc_id, assoc_rev = self.RR.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertNotEqual(assoc_id, None) find_assoc = self.RR.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(find_assoc[0]._id == assoc_id) subj = self.RR.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) res_obj1 = self.RR.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) self.RR.delete_association(assoc_id) self.RR.delete_attachment(att_id) self.RR.delete(resource_id)
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient self.processdispatchclient = ProcessDispatcherServiceClient(node = self.container.node) def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.info('test_createTransformsThenActivateInstrument: new InstrumentModel id = %s', instModel_id) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s', instDevice_id) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance" ) self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) log.info( 'TestDataProcessWithLookupTable: new Stream Definition id = %s', instDevice_id) log.info( 'Creating new CDM data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) log.info( 'new ctd_parsed_data_product_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.info('TestDataProcessWithLookupTable: Data product streams1 = %s', stream_ids) #------------------------------- # Create CTD Raw as the second data product #------------------------------- log.info('TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition') raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='ctd_raw', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) log.info( 'new ctd_raw_data_product_id = %s', ctd_raw_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.info( 'Data product streams2 = %s', stream_ids) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) ) processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) ) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) out_prods = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id,ctd_l0_temperature_output_dp_id] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L0_all_dprocdef_id, in_data_product_ids = in_prods, out_data_product_ids = out_prods) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProcessWithLookupTable: create L0 all data_process return") data_process= self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject= ctd_l0_all_data_process_id, predicate= PRED.hasProcess, object_type= RT.Process,id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s', processAttachment)
class TestIntDataAcquisitionManagementService(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataAcquisitionManagementService self.client = DataAcquisitionManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) def tearDown(self): pass #@unittest.skip('Not done yet.') def test_data_source_ops(self): # test creating a new data source print 'Creating new data source' datasource_obj = IonObject(RT.DataSource, name='DataSource1', description='instrument based new source' , data_source_type='sbe37') try: ds_id = self.client.create_data_source(datasource_obj) except BadRequest as ex: self.fail("failed to create new data source: %s" %ex) print 'new data source id = ', ds_id # test reading a non-existent data source print 'reading non-existent data source' try: dp_obj = self.client.read_data_source('some_fake_id') except NotFound as ex: pass else: self.fail("non-existing data source was found during read: %s" %dp_obj) # update a data source (tests read also) print 'Updating data source' # first get the existing data source object try: datasource_obj = self.client.read_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass # now tweak the object datasource_obj.description = 'the very first data source' # now write the dp back to the registry try: update_result = self.client.update_data_source(datasource_obj) except NotFound as ex: self.fail("existing data source was not found during update") except Conflict as ex: self.fail("revision conflict exception during data source update") #else: # self.assertTrue(update_result == True) # now get the data source back to see if it was updated try: datasource_obj = self.client.read_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass self.assertTrue(datasource_obj.description == 'the very first data source') # now 'delete' the data source print "deleting data source" try: delete_result = self.client.force_delete_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during delete") #self.assertTrue(delete_result == True) # now try to get the deleted dp object try: dp_obj = self.client.read_data_source(ds_id) except NotFound as ex: pass else: self.fail("deleted data source was found during read") # now try to delete the already deleted data source object print "deleting non-existing data source" try: delete_result = self.client.delete_data_source(ds_id) except NotFound as ex: pass else: self.fail("non-existing data source was found during delete") def test_register_instrument(self): # set up initial instrument to register instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) self.base_register_instrument(instrument_id) def test_register_platform(self): # set up initial instrument to register platform_obj = IonObject(RT.PlatformDevice, name='Plat1',description='a platform that is creating the data product') platform_id, rev = self.rrclient.create(platform_obj) #@unittest.skip('Not done yet.') def base_register_instrument(self, instrument_id): # Register an instrument as a data producer in coordination with DM PubSub: create stream, register and create producer object dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new data producer try: ds_id = self.client.register_instrument(instrument_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) print 'new data producer id = ', ds_id # test assigning a data product to an instrument, creating the stream for the product try: self.client.assign_data_product(instrument_id, dataproduct_id) self.client.assign_data_product_source(dataproduct_id, instrument_id) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" %ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" %ex) assocs = self.rrclient.find_associations(dataproduct_id, PRED.hasSource, instrument_id) if not assocs or len(assocs) == 0: self.fail("failed to assign data product to data producer") # test UNassigning a data product from instrument, deleting the stream for the product try: self.client.unassign_data_product(instrument_id, dataproduct_id) self.client.unassign_data_product_source(dataproduct_id, instrument_id) except BadRequest as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) except NotFound as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) assocs = self.rrclient.find_associations(dataproduct_id, PRED.hasSource, instrument_id) if assocs: self.fail("failed to unassign data product to data producer") # test UNregistering a new data producer try: ds_id = self.client.unregister_instrument(instrument_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" %ex) def test_register_external_data_set(self): # Register an external data set as a data producer in coordination with DM PubSub: create stream, register and create producer object # set up initial instrument to register ext_dataset_obj = IonObject(RT.ExternalDataset, name='DataSet1',description='an external data feed') ext_dataset_id, rev = self.rrclient.create(ext_dataset_obj) dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new external data set try: ds_id = self.client.register_external_data_set(ext_dataset_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) print 'new data producer id = ', ds_id # test assigning a data product to an ext_dataset_id, creating the stream for the product try: self.client.assign_data_product(ext_dataset_id, dataproduct_id) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" %ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" %ex) # test UNassigning a data product from ext_dataset_id, deleting the stream for the product try: self.client.unassign_data_product(ext_dataset_id, dataproduct_id) except BadRequest as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) except NotFound as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) # test UNregistering a external data set try: ds_id = self.client.unregister_external_data_set(ext_dataset_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" %ex) #@unittest.skip('not ready') def test_eoi_resources(self): # # test creating a new data provider # print 'Creating new external_data_provider' dataprovider_obj = IonObject(RT.ExternalDataProvider, name='ExtDataProvider1', description='external data provider ') try: dataprovider_id = self.client.create_external_data_provider(dataprovider_obj) except BadRequest as ex: self.fail("failed to create new data provider: %s" %ex) print 'new data provider id = ', dataprovider_id # # test creating a new data source # print 'Creating new data source' datasource_obj = IonObject(RT.DataSource, name='DataSource1', description='data source ', data_source_type='DAP') try: datasource_id = self.client.create_data_source(datasource_obj) except BadRequest as ex: self.fail("failed to create new data source: %s" %ex) print 'new data source id = ', datasource_id # # test creating a new data source model # print 'Creating new data source model' datamodel_obj = IonObject(RT.DataSourceModel, name='DataSourceModel1', description='data source model') try: datamodel_id = self.client.create_data_source_model(datamodel_obj) except BadRequest as ex: self.fail("failed to create new data source model: %s" %ex) print 'new data source model id = ', datamodel_id # # test creating a new external data set # print 'Creating new external data set' dataset_obj = IonObject(RT.ExternalDataset, name='ExternalDataSet1', description='external data set ') try: extdataset_id = self.client.create_external_dataset(dataset_obj) except BadRequest as ex: self.fail("failed to create new external data set: %s" %ex) print 'new external data set id = ', extdataset_id # # test creating a new dataset agent instance # print 'Creating new external data agent ' datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent ') try: datasetagent_id = self.client.create_external_dataset_agent(datasetagent_obj) except BadRequest as ex: self.fail("failed to create new external dataset agent: %s" %ex) print 'new external data agent id = ', datasetagent_id # # test creating a new datasource agent instance # print 'Creating new data source agent ' datasourceagent_obj = IonObject(RT.DataSourceAgent, name='DataSourceAgent1', description=' DataSource agent ') try: datasource_agent_id = self.client.create_data_source_agent(datasourceagent_obj) except BadRequest as ex: self.fail("failed to create new external datasource agent: %s" %ex) print 'new external data agent id = ', datasource_agent_id # # test creating a new dataset agent instance # print 'Creating new external dataset agent instance' datasetagentinstance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external dataset agent instance ') try: datasetagentinstance_id = self.client.create_external_dataset_agent_instance(datasetagentinstance_obj, datasetagent_id) except BadRequest as ex: self.fail("failed to create new external dataset agent instance: %s" %ex) print 'new external data agent instance id = ', datasetagentinstance_id # # test creating a new datasource agent instance # print 'Creating new data source agent ' datasourceagentinstance_obj = IonObject(RT.DataSourceAgentInstance, name='ExternalDataSourceAgentInstance1', description='external DataSource agent instance ') try: datasource_agent_instance_id = self.client.create_data_source_agent_instance(datasourceagentinstance_obj) except BadRequest as ex: self.fail("failed to create new external datasource agent instance: %s" %ex) print 'new external data agent id = ', datasource_agent_instance_id # # test assign / unassign # self.client.unassign_data_source_from_external_data_provider(datasource_id, dataprovider_id) self.client.unassign_data_source_from_data_model(datasource_id, datamodel_id) self.client.unassign_external_dataset_from_data_source(extdataset_id, datasource_id) # # test read # try: dp_obj = self.client.read_external_data_provider(dataprovider_id) except NotFound as ex: self.fail("existing data provicer was not found during read") else: pass try: dp_obj = self.client.read_data_source(datasource_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass # # test delete # try: self.client.delete_external_data_provider(dataprovider_id) self.client.delete_data_source(datasource_id) self.client.delete_external_dataset(extdataset_id) self.client.delete_data_source_model(datamodel_id) self.client.delete_external_dataset_agent(datasetagent_id) self.client.delete_data_source_agent_instance(datasource_agent_instance_id) self.client.force_delete_external_data_provider(dataprovider_id) self.client.force_delete_data_source(datasource_id) self.client.force_delete_external_dataset(extdataset_id) self.client.force_delete_data_source_model(datamodel_id) self.client.force_delete_external_dataset_agent(datasetagent_id) self.client.force_delete_data_source_agent_instance(datasource_agent_instance_id) except NotFound as ex: self.fail("existing data product was not found during delete") # test reading a non-existent data product print 'reading non-existent data product' try: bad_obj = self.client.read_external_data_provider('some_fake_id') except NotFound as ex: pass else: self.fail("non-existing data product was found during read: %s" %bad_obj) def make_grt_parser(self): return self.client.create_parser(Parser(name='grt', description='', module='ion.util.parsers.global_range_test', method='grt_parser', config=None)) @unittest.skip("Deprecated") def test_qc_attachment(self): instrument_device = InstrumentDevice(name='whatever') instrument_device_id,_ = self.rrclient.create(instrument_device) self.addCleanup(self.rrclient.delete, instrument_device_id) self.client.register_instrument(instrument_device_id) self.addCleanup(self.client.unregister_instrument, instrument_device_id) dp = DataProduct(name='instrument output') dp_id,_ = self.rrclient.create(dp) self.addCleanup(self.rrclient.delete, dp_id) parser_id = self.make_grt_parser() attachment = Attachment(name='qc ref', attachment_type=AttachmentType.REFERENCE,content=global_range_test_document, context=ReferenceAttachmentContext(parser_id=parser_id)) att_id = self.rrclient.create_attachment(dp_id, attachment) self.addCleanup(self.rrclient.delete_attachment, att_id) attachment2 = Attachment(name='qc ref2', attachment_type=AttachmentType.REFERENCE, content=global_range_test_document2, context=ReferenceAttachmentContext(parser_id=parser_id)) att2_id = self.rrclient.create_attachment(dp_id, attachment2) self.addCleanup(self.rrclient.delete_attachment, att2_id) self.client.assign_data_product(instrument_device_id, dp_id) self.addCleanup(self.client.unassign_data_product, instrument_device_id, dp_id) svm = StoredValueManager(self.container) doc = svm.read_value('grt_CE01ISSM-MF005-01-CTDBPC999_TEMPWAT') np.testing.assert_array_almost_equal(doc['grt_min_value'], -2.)
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() # create missing data process definition dpd_obj = IonObject(RT.DataProcessDefinition, name=LOGICAL_TRANSFORM_DEFINITION_NAME, description="normally in preload", module='ion.processes.data.transforms.logical_transform', class_name='logical_transform') self.dataprocessclient.create_data_process_definition(dpd_obj) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip('not ready') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [parsed_config] ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) self.assertEqual(1, len(extended_product.data_processes) ) self.assertEqual(3, len(extended_product.process_input_data_products) ) # log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) # log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id) #------------------------------- # Cleanup #------------------------------- self.dpmsclient.delete_data_product(ctd_parsed_data_product) self.dpmsclient.delete_data_product(log_data_product_id) self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() #@unittest.skip('not ready') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", stream_configuration= {'parsed': 'ctd_parsed_param_dict' } ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4003, 'data_port': 4000, 'log_level': 5, } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) #self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.provenance_product_list.status) log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) #log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(1, len(extended_process_def.data_products) ) #one list because of one data process self.assertEqual(3, len(extended_process_def.data_products[0]) ) #inside that inner list are the three output data products #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
class TestResourceRegistry(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url("res/deploy/r2coi.yml") # Now create client to bank service self.resource_registry_service = ResourceRegistryServiceClient(node=self.container.node) def test_crud(self): # Some quick registry tests # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", name="name", foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name", "foo": "bar"}) self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name"}, foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Instantiate an object obj = IonObject("UserInfo", name="name") # Can set attributes that aren't in the object's schema with self.assertRaises(AttributeError) as cm: setattr(obj, "foo", "bar") self.assertTrue(cm.exception.message == "'UserInfo' object has no attribute 'foo'") # Cam't call update with object that hasn't been persisted with self.assertRaises(BadRequest) as cm: self.resource_registry_service.update(obj) self.assertTrue(cm.exception.message.startswith("Object does not have required '_id' or '_rev' attribute")) # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) # Cannot create object with _id and _rev fields pre-set with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create(read_obj) self.assertTrue(cm.exception.message.startswith("Doc must not have '_id'")) # Update object read_obj.name = "John Doe" self.resource_registry_service.update(read_obj) # Update should fail with revision mismatch with self.assertRaises(Conflict) as cm: self.resource_registry_service.update(read_obj) self.assertTrue(cm.exception.message.startswith("Object not based on most current version")) # Re-read and update object read_obj = self.resource_registry_service.read(obj_id) self.resource_registry_service.update(read_obj) # Delete object self.resource_registry_service.delete(obj_id) # Make sure read, update and delete report error with self.assertRaises(NotFound) as cm: self.resource_registry_service.read(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.update(read_obj) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) # Owner creation tests user = IonObject("ActorIdentity", name="user") uid, _ = self.resource_registry_service.create(user) inst = IonObject("InstrumentDevice", name="instrument") iid, _ = self.resource_registry_service.create(inst, headers={"ion-actor-id": str(uid)}) ids, _ = self.resource_registry_service.find_objects(iid, PRED.hasOwner, RT.ActorIdentity, id_only=True) self.assertEquals(len(ids), 1) assoc = self.resource_registry_service.read(ids[0]) self.resource_registry_service.delete(iid) with self.assertRaises(NotFound) as ex: assoc = self.resource_registry_service.read(iid) def test_lifecycle(self): att = IonObject("InstrumentDevice", name="mine", description="desc") rid, rev = self.resource_registry_service.create(att) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.name, att.name) self.assertEquals(att1.lcstate, LCS.DRAFT_PRIVATE) new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.PLAN) self.assertEquals(new_state, LCS.PLANNED_PRIVATE) att2 = self.resource_registry_service.read(rid) self.assertEquals(att2.lcstate, LCS.PLANNED_PRIVATE) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.execute_lifecycle_transition(rid, LCE.UNANNOUNCE) self.assertTrue( "type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce" in cm.exception.message ) new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.DEVELOP) self.assertEquals(new_state, LCS.DEVELOPED_PRIVATE) self.assertRaises( iex.BadRequest, self.resource_registry_service.execute_lifecycle_transition, resource_id=rid, transition_event="NONE##", ) self.resource_registry_service.set_lifecycle_state(rid, LCS.INTEGRATED_PRIVATE) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.lcstate, LCS.INTEGRATED_PRIVATE) def test_association(self): # Instantiate ActorIdentity object actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create(actor_identity_obj) read_actor_identity_obj = self.resource_registry_service.read(actor_identity_obj_id) # Instantiate UserInfo object user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create(user_info_obj) read_user_info_obj = self.resource_registry_service.read(user_info_obj_id) # Test create failures with self.assertRaises(AttributeError) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.bogus, user_info_obj_id) self.assertTrue(cm.exception.message == "bogus") # Predicate not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Association must have all elements set") # Bad association type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, "bogustype" ) self.assertTrue(cm.exception.message == "Unsupported assoc_type: bogustype") # Subject id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(None, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Association must have all elements set") # Object id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, None) self.assertTrue(cm.exception.message == "Association must have all elements set") # Bad subject id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association("bogus", PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Object with id bogus does not exist.") # Bad object id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, "bogus") self.assertTrue(cm.exception.message == "Object with id bogus does not exist.") # _id missing from subject with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Subject id or rev not available") # _id missing from object with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj) self.assertTrue(cm.exception.message == "Object id or rev not available") # Wrong subject type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(user_info_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal subject type UserInfo for predicate hasInfo") # Wrong object type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id ) self.assertTrue(cm.exception.message == "Illegal object type ActorIdentity for predicate hasInfo") # Create two different association types between the same subject and predicate assoc_id1, assoc_rev1 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id ) # Read object, subject res_obj1 = self.resource_registry_service.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True ) self.assertEquals(res_obj1, user_info_obj_id) res_obj2 = self.resource_registry_service.read_subject(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id) self.assertEquals(res_obj2._id, actor_identity_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True ) self.assertEquals(res_obj2, actor_identity_obj_id) # Create a similar association to a specific revision # TODO: This is not a supported case so far assoc_id2, assoc_rev2 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, "H2R" ) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False ) ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=False) ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=False) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo, read_user_info_obj ) ret2 = self.resource_registry_service.find_associations(read_actor_identity_obj, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None, True ) ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=True) ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=True) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(None, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(actor_identity_obj_id, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(None, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(actor_identity_obj, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Object id not available in subject") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(actor_identity_obj_id, None, user_info_obj) self.assertTrue(cm.exception.message == "Object id not available in object") # Find subjects (good cases) subj_ret1 = self.resource_registry_service.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) subj_ret2 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True ) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, True) subj_ret4 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, False) subj_ret6 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find subjects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(None, None, None) self.assertTrue(cm.exception.message == "Must provide object") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.bogus, user_info_obj_id, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_subjects(RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj, True) self.assertTrue(cm.exception.message == "Object id not available in object") # Find objects (good cases) subj_ret1 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True) subj_ret2 = self.resource_registry_service.find_objects( read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True ) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, True) subj_ret4 = self.resource_registry_service.find_objects(actor_identity_obj_id, None, None, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, False) subj_ret6 = self.resource_registry_service.find_objects(read_actor_identity_obj, None, None, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find objects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(None, None, None) self.assertTrue(cm.exception.message == "Must provide subject") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True ) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue(cm.exception.message == "Object id not available in subject") # Get association (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(None, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(actor_identity_obj_id, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(None, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(actor_identity_obj, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Object id not available in subject") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(actor_identity_obj_id, None, user_info_obj) self.assertTrue(cm.exception.message == "Object id not available in object") # Delete one of the associations self.resource_registry_service.delete_association(assoc_id2) assoc = self.resource_registry_service.get_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(assoc._id == assoc_id1) # Delete (bad cases) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete_association("bogus") self.assertTrue(cm.exception.message == "Object with id bogus does not exist.") # Delete other association self.resource_registry_service.delete_association(assoc_id1) # Delete resources self.resource_registry_service.delete(actor_identity_obj_id) self.resource_registry_service.delete(user_info_obj_id) def test_find_resources(self): with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_resources(RT.UserInfo, LCS.DRAFT, "name", False) self.assertTrue(cm.exception.message == "find by name does not support lcstate") ret = self.resource_registry_service.find_resources(RT.UserInfo, None, "name", False) self.assertTrue(len(ret[0]) == 0) # Instantiate an object obj = IonObject("InstrumentDevice", name="name") # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) ret = self.resource_registry_service.find_resources(RT.InstrumentDevice, None, "name", False) self.assertTrue(len(ret[0]) == 1) self.assertTrue(ret[0][0]._id == read_obj._id) ret = self.resource_registry_service.find_resources(RT.InstrumentDevice, LCS.DRAFT, None, False) self.assertTrue(len(ret[0]) == 1) self.assertTrue(ret[0][0]._id == read_obj._id) # @attr('INT', group='coirr1') # class TestResourceRegistry1(IonIntegrationTestCase): # # def setUp(self): # # Start container # self._start_container() # self.container.start_rel_from_url('res/deploy/r2coi.yml') # # # Now create client to bank service # self.resource_registry_service = ResourceRegistryServiceClient(node=self.container.node) def test_attach(self): binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82" # Owner creation tests instrument = IonObject("InstrumentDevice", name="instrument") iid, _ = self.resource_registry_service.create(instrument) att = Attachment(content=binary, attachment_type=AttachmentType.BLOB) aid1 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid1) self.assertEquals(binary, att1.content) import base64 att = Attachment(content=base64.encodestring(binary), attachment_type=AttachmentType.ASCII) aid2 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid2) self.assertEquals(binary, base64.decodestring(att1.content)) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid1, aid2]) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True) self.assertEquals(att_ids, [aid2, aid1]) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True, limit=1) self.assertEquals(att_ids, [aid2]) atts = self.resource_registry_service.find_attachments(iid, id_only=False, limit=1) self.assertEquals(atts[0].content, att1.content) self.resource_registry_service.delete_attachment(aid1) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid2]) self.resource_registry_service.delete_attachment(aid2) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [])
class TestTransformWorker(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Instantiate a process to represent the test process = TransformWorkerTestProcess() self.dataset_management_client = DatasetManagementServiceClient( node=self.container.node) self.pubsub_client = PubsubManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceProcessClient( node=self.container.node, process=process) self.time_dom, self.spatial_dom = time_series_domain() self.ph = ParameterHelper(self.dataset_management_client, self.addCleanup) self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10) def push_granule(self, data_product_id): ''' Publishes and monitors that the granule arrived ''' datasets, _ = self.rrclient.find_objects(data_product_id, PRED.hasDataset, id_only=True) dataset_monitor = DatasetMonitor(datasets[0]) rdt = self.ph.rdt_for_data_product(data_product_id) self.ph.fill_parsed_rdt(rdt) self.ph.publish_rdt_to_data_product(data_product_id, rdt) assert dataset_monitor.wait() dataset_monitor.stop() @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_transform_worker(self): # test that a data process (type: data-product-in / data-product-out) can be defined and launched. # verify that the output granule fields are correctly populated # test that the input and output data products are linked to facilitate provenance self.dp_list = [] self.data_process_objs = [] self._output_stream_ids = [] self.granule_verified = Event() self.worker_assigned_event_verified = Event() self.dp_created_event_verified = Event() self.heartbeat_event_verified = Event() self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name( name='ctd_parsed_param_dict', id_only=True) # create the StreamDefinition self.stream_def_id = self.pubsub_client.create_stream_definition( name='stream_def', parameter_dictionary_id=self.parameter_dict_id) self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id) # create the DataProduct that is the input to the data processes input_dp_obj = IonObject(RT.DataProduct, name='input_data_product', description='input test stream') self.input_dp_id = self.dataproductclient.create_data_product( data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # retrieve the Stream for this data product stream_ids, assoc_ids = self.rrclient.find_objects( self.input_dp_id, PRED.hasStream, RT.Stream, True) self.stream_id = stream_ids[0] self.start_event_listener() # create the DPD, DataProcess and output DataProduct dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process( ) self.dp_list.append(dataprocess_id) # validate the repository for data product algorithms persists the new resources NEW SA-1 # create_data_process call created one of each dpd_ids, _ = self.rrclient.find_resources( restype=OT.DataProcessDefinition, id_only=False) # there will be more than one becuase of the DPDs that reperesent the PFs in the data product above self.assertTrue(dpd_ids is not None) dp_ids, _ = self.rrclient.find_resources(restype=OT.DataProcess, id_only=False) # only one DP becuase the PFs that are in the code dataproduct above are not activated yet. self.assertEquals(len(dp_ids), 1) # validate the name and version label NEW SA - 2 dataprocessdef_obj = self.dataprocessclient.read_data_process_definition( dataprocessdef_id) self.assertEqual(dataprocessdef_obj.version_label, '1.0a') self.assertEqual(dataprocessdef_obj.name, 'add_arrays') # validate that the DPD has an attachment NEW SA - 21 attachment_ids, assoc_ids = self.rrclient.find_objects( dataprocessdef_id, PRED.hasAttachment, RT.Attachment, True) self.assertEqual(len(attachment_ids), 1) attachment_obj = self.rrclient.read_attachment(attachment_ids[0]) log.debug('attachment: %s', attachment_obj) # validate that the data process resource has input and output data products associated # L4-CI-SA-RQ-364 and NEW SA-3 outproduct_ids, assoc_ids = self.rrclient.find_objects( dataprocess_id, PRED.hasOutputProduct, RT.DataProduct, True) self.assertEqual(len(outproduct_ids), 1) inproduct_ids, assoc_ids = self.rrclient.find_objects( dataprocess_id, PRED.hasInputProduct, RT.DataProduct, True) self.assertEqual(len(inproduct_ids), 1) # Test for provenance. Get Data product produced by the data processes output_data_product_id, _ = self.rrclient.find_objects( subject=dataprocess_id, object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, id_only=True) output_data_product_provenance = self.dataproductclient.get_data_product_provenance( output_data_product_id[0]) # Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the # DataProcessDefinition creating the child from the parent. self.assertTrue(len(output_data_product_provenance) == 2) self.assertTrue(self.input_dp_id in output_data_product_provenance[ output_data_product_id[0]]['parents']) self.assertTrue(output_data_product_provenance[ output_data_product_id[0]]['parents'][self.input_dp_id] ['data_process_definition_id'] == dataprocessdef_id) # NEW SA - 4 | Data processing shall include the appropriate data product algorithm name and version number in # the metadata of each output data product created by the data product algorithm. output_data_product_obj, _ = self.rrclient.find_objects( subject=dataprocess_id, object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, id_only=False) self.assertTrue(output_data_product_obj[0].name != None) self.assertTrue(output_data_product_obj[0]._rev != None) # retrieve subscription from data process subscription_objs, _ = self.rrclient.find_objects( subject=dataprocess_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False) log.debug('test_transform_worker subscription_obj: %s', subscription_objs[0]) # create a queue to catch the published granules self.subscription_id = self.pubsub_client.create_subscription( name='parsed_subscription', stream_ids=[self.stream_id], exchange_name=subscription_objs[0].exchange_name) self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id) self.pubsub_client.activate_subscription(self.subscription_id) self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id) stream_route = self.pubsub_client.read_stream_route(self.stream_id) self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id, stream_route=stream_route) for n in range(1, 101): rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) rdt['time'] = [0] # time should always come first rdt['conductivity'] = [1] rdt['pressure'] = [2] rdt['salinity'] = [8] self.publisher.publish(rdt.to_granule()) # validate that the output granule is received and the updated value is correct self.assertTrue(self.granule_verified.wait(self.wait_time)) # validate that the data process loaded into worker event is received (L4-CI-SA-RQ-182) self.assertTrue( self.worker_assigned_event_verified.wait(self.wait_time)) # validate that the data process create (with data product ids) event is received (NEW SA -42) self.assertTrue(self.dp_created_event_verified.wait(self.wait_time)) # validate that the data process heartbeat event is received (for every hundred granules processed) (L4-CI-SA-RQ-182) #this takes a while so set wait limit to large value self.assertTrue(self.heartbeat_event_verified.wait(200)) # validate that the code from the transform function can be retrieve via inspect_data_process_definition src = self.dataprocessclient.inspect_data_process_definition( dataprocessdef_id) self.assertIn('def add_arrays(a, b)', src) # now delete the DPD and DP then verify that the resources are retired so that information required for provenance are still available self.dataprocessclient.delete_data_process(dataprocess_id) self.dataprocessclient.delete_data_process_definition( dataprocessdef_id) in_dp_objs, _ = self.rrclient.find_objects( subject=dataprocess_id, predicate=PRED.hasInputProduct, object_type=RT.DataProduct, id_only=True) self.assertTrue(in_dp_objs is not None) dpd_objs, _ = self.rrclient.find_subjects( subject_type=RT.DataProcessDefinition, predicate=PRED.hasDataProcess, object=dataprocess_id, id_only=True) self.assertTrue(dpd_objs is not None) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_transform_worker_with_instrumentdevice(self): # test that a data process (type: data-product-in / data-product-out) can be defined and launched. # verify that the output granule fields are correctly populated # test that the input and output data products are linked to facilitate provenance self.data_process_objs = [] self._output_stream_ids = [] self.event_verified = Event() # Create CTD Parsed as the initial data product # create a stream definition for the data from the ctd simulator self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) self.stream_def_id = self.pubsub_client.create_stream_definition( name='stream_def', parameter_dictionary_id=self.parameter_dict_id) # create the DataProduct that is the input to the data processes input_dp_obj = IonObject(RT.DataProduct, name='input_data_product', description='input test stream') self.input_dp_id = self.dataproductclient.create_data_product( data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # retrieve the Stream for this data product stream_ids, assoc_ids = self.rrclient.find_objects( self.input_dp_id, PRED.hasStream, RT.Stream, True) self.stream_id = stream_ids[0] log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id) # only ever need one device for testing purposes. instDevice_obj, _ = self.rrclient.find_resources( restype=RT.InstrumentDevice, name='test_ctd_device') if instDevice_obj: instDevice_id = instDevice_obj[0]._id else: instDevice_obj = IonObject(RT.InstrumentDevice, name='test_ctd_device', description="test_ctd_device", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=self.input_dp_id) # create the DPD, DataProcess and output DataProduct dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process( ) self.addCleanup(self.dataprocessclient.delete_data_process, dataprocess_id) self.addCleanup(self.dataprocessclient.delete_data_process_definition, dataprocessdef_id) # Test for provenance. Get Data product produced by the data processes output_data_product_id, _ = self.rrclient.find_objects( subject=dataprocess_id, object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, id_only=True) output_data_product_provenance = self.dataproductclient.get_data_product_provenance( output_data_product_id[0]) # Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the # DataProcessDefinition creating the child from the parent. self.assertTrue(len(output_data_product_provenance) == 3) self.assertTrue(self.input_dp_id in output_data_product_provenance[ output_data_product_id[0]]['parents']) self.assertTrue(instDevice_id in output_data_product_provenance[ self.input_dp_id]['parents']) self.assertTrue(output_data_product_provenance[instDevice_id]['type'] == 'InstrumentDevice') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_transform_worker_with_platformdevice(self): # test that a data process (type: data-product-in / data-product-out) can be defined and launched. # verify that the output granule fields are correctly populated # test that the input and output data products are linked to facilitate provenance self.data_process_objs = [] self._output_stream_ids = [] self.event_verified = Event() # Create CTD Parsed as the initial data product # create a stream definition for the data from the ctd simulator self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) self.stream_def_id = self.pubsub_client.create_stream_definition( name='stream_def', parameter_dictionary_id=self.parameter_dict_id) # create the DataProduct that is the input to the data processes input_dp_obj = IonObject(RT.DataProduct, name='input_data_product', description='input test stream') self.input_dp_id = self.dataproductclient.create_data_product( data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # retrieve the Stream for this data product stream_ids, assoc_ids = self.rrclient.find_objects( self.input_dp_id, PRED.hasStream, RT.Stream, True) self.stream_id = stream_ids[0] log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id) # only ever need one device for testing purposes. platform_device_obj, _ = self.rrclient.find_resources( restype=RT.PlatformDevice, name='TestPlatform') if platform_device_obj: platform_device_id = platform_device_obj[0]._id else: platform_device_obj = IonObject(RT.PlatformDevice, name='TestPlatform', description="TestPlatform", serial_number="12345") platform_device_id = self.imsclient.create_platform_device( platform_device=platform_device_obj) self.damsclient.assign_data_product( input_resource_id=platform_device_id, data_product_id=self.input_dp_id) # create the DPD, DataProcess and output DataProduct dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process( ) self.addCleanup(self.dataprocessclient.delete_data_process, dataprocess_id) self.addCleanup(self.dataprocessclient.delete_data_process_definition, dataprocessdef_id) # Test for provenance. Get Data product produced by the data processes output_data_product_id, _ = self.rrclient.find_objects( subject=dataprocess_id, object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, id_only=True) output_data_product_provenance = self.dataproductclient.get_data_product_provenance( output_data_product_id[0]) # Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the # DataProcessDefinition creating the child from the parent. self.assertTrue(len(output_data_product_provenance) == 3) self.assertTrue(self.input_dp_id in output_data_product_provenance[ output_data_product_id[0]]['parents']) self.assertTrue(platform_device_id in output_data_product_provenance[ self.input_dp_id]['parents']) self.assertTrue(output_data_product_provenance[platform_device_id] ['type'] == 'PlatformDevice') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_event_transform_worker(self): self.data_process_objs = [] self._output_stream_ids = [] self.event_verified = Event() # test that a data process (type: data-product-in / event-out) can be defined and launched. # verify that event fields are correctly populated self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name( name='ctd_parsed_param_dict', id_only=True) # create the StreamDefinition self.stream_def_id = self.pubsub_client.create_stream_definition( name='stream_def', parameter_dictionary_id=self.parameter_dict_id) self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id) # create the DataProduct input_dp_obj = IonObject(RT.DataProduct, name='input_data_product', description='input test stream') self.input_dp_id = self.dataproductclient.create_data_product( data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # retrieve the Stream for this data product stream_ids, assoc_ids = self.rrclient.find_objects( self.input_dp_id, PRED.hasStream, RT.Stream, True) self.stream_id = stream_ids[0] # create the DPD and two DPs self.event_data_process_id = self.create_event_data_processes() # retrieve subscription from data process subscription_objs, _ = self.rrclient.find_objects( subject=self.event_data_process_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False) log.debug('test_event_transform_worker subscription_obj: %s', subscription_objs[0]) # create a queue to catch the published granules self.subscription_id = self.pubsub_client.create_subscription( name='parsed_subscription', stream_ids=[self.stream_id], exchange_name=subscription_objs[0].exchange_name) self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id) self.pubsub_client.activate_subscription(self.subscription_id) self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id) stream_route = self.pubsub_client.read_stream_route(self.stream_id) self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id, stream_route=stream_route) self.start_event_transform_listener() self.data_modified = Event() rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) rdt['time'] = [0] # time should always come first rdt['conductivity'] = [1] rdt['pressure'] = [2] rdt['salinity'] = [8] self.publisher.publish(rdt.to_granule()) self.assertTrue(self.event_verified.wait(self.wait_time)) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_bad_argument_map(self): self._output_stream_ids = [] # test that a data process (type: data-product-in / data-product-out) parameter mapping it validated during # data process creation and that the correct exception is raised for both input and output. self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name( name='ctd_parsed_param_dict', id_only=True) # create the StreamDefinition self.stream_def_id = self.pubsub_client.create_stream_definition( name='stream_def', parameter_dictionary_id=self.parameter_dict_id) self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id) # create the DataProduct that is the input to the data processes input_dp_obj = IonObject(RT.DataProduct, name='input_data_product', description='input test stream') self.input_dp_id = self.dataproductclient.create_data_product( data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # two data processes using one transform and one DPD dp1_func_output_dp_id = self.create_output_data_product() # Set up DPD and DP #2 - array add function tf_obj = IonObject( RT.TransformFunction, name='add_array_func', description='adds values in an array', function='add_arrays', module="ion_example.add_arrays", arguments=['arr1', 'arr2'], function_type=TransformFunctionType.TRANSFORM, uri= 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg' ) add_array_func_id, rev = self.rrclient.create(tf_obj) dpd_obj = IonObject( RT.DataProcessDefinition, name='add_arrays', description='adds the values of two arrays', data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS) add_array_dpd_id = self.dataprocessclient.create_data_process_definition( data_process_definition=dpd_obj, function_id=add_array_func_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.stream_def_id, add_array_dpd_id, binding='add_array_func') # create the data process with invalid argument map argument_map = {"arr1": "foo", "arr2": "bar"} output_param = "salinity" with self.assertRaises(BadRequest) as cm: dp1_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id], outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param) ex = cm.exception log.debug(' exception raised: %s', cm) self.assertEqual( ex.message, "Input data product does not contain the parameters defined in argument map" ) # create the data process with invalid output parameter name argument_map = {"arr1": "conductivity", "arr2": "pressure"} output_param = "foo" with self.assertRaises(BadRequest) as cm: dp1_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id], outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param) ex = cm.exception log.debug(' exception raised: %s', cm) self.assertEqual( ex.message, "Output data product does not contain the output parameter name provided" ) def create_event_data_processes(self): # two data processes using one transform and one DPD argument_map = {"a": "salinity"} # set up DPD and DP #2 - array add function tf_obj = IonObject( RT.TransformFunction, name='validate_salinity_array', description='validate_salinity_array', function='validate_salinity_array', module="ion.processes.data.transforms.test.test_transform_worker", arguments=['a'], function_type=TransformFunctionType.TRANSFORM) add_array_func_id, rev = self.rrclient.create(tf_obj) dpd_obj = IonObject( RT.DataProcessDefinition, name='validate_salinity_array', description='validate_salinity_array', data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS, ) add_array_dpd_id = self.dataprocessclient.create_data_process_definition( data_process_definition=dpd_obj, function_id=add_array_func_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.stream_def_id, add_array_dpd_id, binding='validate_salinity_array') # create the data process dp1_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id], outputs=None, argument_map=argument_map) self.damsclient.register_process(dp1_data_process_id) self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id) return dp1_data_process_id def create_data_process(self): # two data processes using one transform and one DPD dp1_func_output_dp_id = self.create_output_data_product() argument_map = {"arr1": "conductivity", "arr2": "pressure"} output_param = "salinity" # set up DPD and DP #2 - array add function tf_obj = IonObject( RT.TransformFunction, name='add_array_func', description='adds values in an array', function='add_arrays', module="ion_example.add_arrays", arguments=['arr1', 'arr2'], function_type=TransformFunctionType.TRANSFORM, uri= 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg' ) add_array_func_id, rev = self.rrclient.create(tf_obj) dpd_obj = IonObject( RT.DataProcessDefinition, name='add_arrays', description='adds the values of two arrays', data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS, version_label='1.0a') add_array_dpd_id = self.dataprocessclient.create_data_process_definition( data_process_definition=dpd_obj, function_id=add_array_func_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.stream_def_id, add_array_dpd_id, binding='add_array_func') # create the data process dp1_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id], outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param) self.damsclient.register_process(dp1_data_process_id) #self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id) # add an attachment object to this DPD to test new SA-21 import msgpack attachment_content = 'foo bar' attachment_obj = IonObject(RT.Attachment, name='test_attachment', attachment_type=AttachmentType.ASCII, content_type='text/plain', content=msgpack.packb(attachment_content)) att_id = self.rrclient.create_attachment(add_array_dpd_id, attachment_obj) self.addCleanup(self.rrclient.delete_attachment, att_id) return add_array_dpd_id, dp1_data_process_id, dp1_func_output_dp_id def create_output_data_product(self): dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition( name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id) dp1_output_dp_obj = IonObject(RT.DataProduct, name='data_process1_data_product', description='output of add array func') dp1_func_output_dp_id = self.dataproductclient.create_data_product( dp1_output_dp_obj, dp1_outgoing_stream_id) self.addCleanup(self.dataproductclient.delete_data_product, dp1_func_output_dp_id) # retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id, PRED.hasStream, None, True) self._output_stream_ids.append(stream_ids[0]) subscription_id = self.pubsub_client.create_subscription( 'validator', data_product_ids=[dp1_func_output_dp_id]) self.addCleanup(self.pubsub_client.delete_subscription, subscription_id) def on_granule(msg, route, stream_id): log.debug('recv_packet stream_id: %s route: %s msg: %s', stream_id, route, msg) self.validate_output_granule(msg, route, stream_id) self.granule_verified.set() validator = StandaloneStreamSubscriber('validator', callback=on_granule) validator.start() self.addCleanup(validator.stop) self.pubsub_client.activate_subscription(subscription_id) self.addCleanup(self.pubsub_client.deactivate_subscription, subscription_id) return dp1_func_output_dp_id def validate_event(self, *args, **kwargs): """ This method is a callback function for receiving DataProcessStatusEvent. """ data_process_event = args[0] log.debug("DataProcessStatusEvent: %s", str(data_process_event.__dict__)) # if data process already created, check origin if self.dp_list: self.assertIn(data_process_event.origin, self.dp_list) # if this is a heartbeat event then 100 granules have been processed if 'data process status update.' in data_process_event.description: self.heartbeat_event_verified.set() else: # else check that this is the assign event if 'Data process assigned to transform worker' in data_process_event.description: self.worker_assigned_event_verified.set() elif 'Data process created for data product' in data_process_event.description: self.dp_created_event_verified.set() def validate_output_granule(self, msg, route, stream_id): self.assertIn(stream_id, self._output_stream_ids) rdt = RecordDictionaryTool.load_from_granule(msg) log.debug('validate_output_granule rdt: %s', rdt) sal_val = rdt['salinity'] np.testing.assert_array_equal(sal_val, np.array([3])) def start_event_listener(self): es = EventSubscriber(event_type=OT.DataProcessStatusEvent, callback=self.validate_event) es.start() self.addCleanup(es.stop) def validate_transform_event(self, *args, **kwargs): """ This method is a callback function for receiving DataProcessStatusEvent. """ status_alert_event = args[0] np.testing.assert_array_equal(status_alert_event.origin, self.stream_id) np.testing.assert_array_equal(status_alert_event.values, np.array([self.event_data_process_id])) log.debug("DeviceStatusAlertEvent: %s", str(status_alert_event.__dict__)) self.event_verified.set() def start_event_transform_listener(self): es = EventSubscriber(event_type=OT.DeviceStatusAlertEvent, callback=self.validate_transform_event) es.start() self.addCleanup(es.stop) def test_download(self): egg_url = 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg' egg_path = TransformWorker.download_egg(egg_url) import pkg_resources pkg_resources.working_set.add_entry(egg_path) from ion_example.add_arrays import add_arrays a = add_arrays(1, 2) self.assertEquals(a, 3)
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url("res/deploy/r2deploy.yml") # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient def test_lookupTableProcessing(self): # ------------------------------- # Create InstrumentModel # ------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) log.info("test_createTransformsThenActivateInstrument: new InstrumentModel id = %s", instModel_id) # ------------------------------- # Create InstrumentAgent # ------------------------------- instAgent_obj = IonObject( RT.InstrumentAgent, name="agent007", description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.info("test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s", instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # ------------------------------- # Create InstrumentDevice and attachment for lookup table # ------------------------------- instDevice_obj = IonObject( RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) log.info("test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s", instDevice_id) contents = "this is the lookup table contents, replace with a file..." att = IonObject( RT.Attachment, name="deviceLookupTable", content=base64.encodestring(contents), keywords=["DataProcessInput"], attachment_type=AttachmentType.ASCII, ) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) log.info("test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s", deviceAttachment) # ------------------------------- # Create InstrumentAgentInstance to hold configuration information # ------------------------------- driver_config = { "dvr_mod": "mi.instrument.seabird.sbe37smb.ooicore.driver", "dvr_cls": "SBE37Driver", "workdir": "/tmp/", } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstance", description="SBE37IMAgentInstance", driver_config=driver_config, comms_device_address="sbe37-simulator.oceanobservatories.org", comms_device_port=4001, port_agent_work_dir="/tmp/", port_agent_delimeter=["<<", ">>"], ) self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) # ------------------------------- # Create CTD Parsed as the first data product # ------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition( name="SBE37_CDM", parameter_dictionary_id=pdict_id ) log.info("TestDataProcessWithLookupTable: new Stream Definition id = %s", instDevice_id) log.info("Creating new CDM data product with a stream definition") # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject( RT.DataProduct, name="ctd_parsed", description="ctd stream test", temporal_domain=tdom, spatial_domain=sdom ) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) log.info("new ctd_parsed_data_product_id = %s", ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.info("TestDataProcessWithLookupTable: Data product streams1 = %s", stream_ids) # ------------------------------- # Create CTD Raw as the second data product # ------------------------------- log.info("TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition") raw_stream_def_id = self.pubsubclient.create_stream_definition( name="SBE37_RAW", parameter_dictionary_id=pdict_id ) dp_obj = IonObject( RT.DataProduct, name="ctd_raw", description="raw stream test", temporal_domain=tdom, spatial_domain=sdom ) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) log.info("new ctd_raw_data_product_id = %s", ctd_raw_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.info("Data product streams2 = %s", stream_ids) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition # ------------------------------- log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all") dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L0_all", description="transform ctd package into three separate L0 streams", module="ion.processes.data.transforms.ctd.ctd_L0_all", class_name="ctd_L0_all", ) try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" % ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject( RT.Attachment, name="processDefinitionLookupTable", content=base64.encodestring(contents), keywords=["DataProcessInput"], attachment_type=AttachmentType.ASCII, ) processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att) log.debug( "TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment), ) processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment) log.debug( "TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj), ) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products # ------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name="L0_Conductivity", parameter_dictionary_id=pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding="conductivity" ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name="L0_Pressure", parameter_dictionary_id=pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding="pressure" ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name="L0_Temperature", parameter_dictionary_id=pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding="temperature" ) self.output_products = {} log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name="L0_Conductivity", description="transform output conductivity", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id ) self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name="L0_Pressure", description="transform output pressure", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id ) self.output_products["pressure"] = ctd_l0_pressure_output_dp_id log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name="L0_Temperature", description="transform output temperature", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id ) self.output_products["temperature"] = ctd_l0_temperature_output_dp_id # ------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process # ------------------------------- log.debug("TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, in_prods, self.output_products ) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug("TestDataProcessWithLookupTable: create L0 all data_process return") contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject( RT.Attachment, name="processLookupTable", content=base64.encodestring(contents), keywords=["DataProcessInput"], attachment_type=AttachmentType.ASCII, ) processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info("TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s", processAttachment)
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) #@unittest.skip('not working') def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.services.mi.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', deviceAttachment #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- driver_config = { 'dvr_mod' : 'ion.services.mi.drivers.sbe37_driver', 'dvr_cls' : 'SBE37Driver', 'workdir' : '/tmp/', } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', instDevice_id print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test') try: ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids #------------------------------- # Create CTD Raw as the second data product #------------------------------- print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition' raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def) dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test') try: ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new ctd_raw_data_product_id = ', ctd_raw_data_product self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) ) processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) ) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products={} log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity') ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure') ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature') ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, ctd_parsed_data_product, self.output_products) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return") contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', processAttachment
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) #@unittest.skip('not working') def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', deviceAttachment #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- driver_config = { 'dvr_mod' : 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls' : 'SBE37Driver', 'workdir' : '/tmp/', } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) print 'TestDataProcessWithLookupTable: new Stream Definition id = ', instDevice_id print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test') try: ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) print 'TestDataProcessWithLookupTable: Data product streams1 = ', stream_ids #------------------------------- # Create CTD Raw as the second data product #------------------------------- print 'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition' raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def) dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test') try: ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new ctd_raw_data_product_id = ', ctd_raw_data_product self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) ) processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) ) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products={} log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity') ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True) log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure') ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True) log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature') ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, in_prods, self.output_products) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProcessWithLookupTable: create L0 all data_process return") contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) print 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = ', processAttachment
class TestResourceRegistry(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2coi.yml') # Now create client to bank service self.resource_registry_service = ResourceRegistryServiceClient( node=self.container.node) def test_crud(self): # Some quick registry tests # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", name="name", foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name", "foo": "bar"}) self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name"}, foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Instantiate an object obj = IonObject("UserInfo", name="name") # Can set attributes that aren't in the object's schema with self.assertRaises(AttributeError) as cm: setattr(obj, "foo", "bar") self.assertTrue( cm.exception.message == "'UserInfo' object has no attribute 'foo'") # Cam't call update with object that hasn't been persisted with self.assertRaises(BadRequest) as cm: self.resource_registry_service.update(obj) self.assertTrue( cm.exception.message.startswith( "Object does not have required '_id' or '_rev' attribute")) # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) # Cannot create object with _id and _rev fields pre-set with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create(read_obj) self.assertTrue( cm.exception.message.startswith("Doc must not have '_id'")) # Update object read_obj.name = "John Doe" self.resource_registry_service.update(read_obj) # Update should fail with revision mismatch with self.assertRaises(Conflict) as cm: self.resource_registry_service.update(read_obj) self.assertTrue( cm.exception.message.startswith( "Object not based on most current version")) # Re-read and update object read_obj = self.resource_registry_service.read(obj_id) self.resource_registry_service.update(read_obj) # Delete object self.resource_registry_service.delete(obj_id) # Make sure read, update and delete report error with self.assertRaises(NotFound) as cm: self.resource_registry_service.read(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.update(read_obj) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) # Owner creation tests user = IonObject("ActorIdentity", name='user') uid, _ = self.resource_registry_service.create(user) inst = IonObject("InstrumentDevice", name='instrument') iid, _ = self.resource_registry_service.create( inst, headers={'ion-actor-id': str(uid)}) ids, _ = self.resource_registry_service.find_objects(iid, PRED.hasOwner, RT.ActorIdentity, id_only=True) self.assertEquals(len(ids), 1) assoc = self.resource_registry_service.read(ids[0]) self.resource_registry_service.delete(iid) with self.assertRaises(NotFound) as ex: assoc = self.resource_registry_service.read(iid) def test_lifecycle(self): att = IonObject("InstrumentDevice", name='mine', description='desc') rid, rev = self.resource_registry_service.create(att) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.name, att.name) self.assertEquals(att1.lcstate, LCS.DRAFT_PRIVATE) new_state = self.resource_registry_service.execute_lifecycle_transition( rid, LCE.PLAN) self.assertEquals(new_state, LCS.PLANNED_PRIVATE) att2 = self.resource_registry_service.read(rid) self.assertEquals(att2.lcstate, LCS.PLANNED_PRIVATE) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.execute_lifecycle_transition( rid, LCE.UNANNOUNCE) self.assertTrue( "type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce" in cm.exception.message) new_state = self.resource_registry_service.execute_lifecycle_transition( rid, LCE.DEVELOP) self.assertEquals(new_state, LCS.DEVELOPED_PRIVATE) self.assertRaises( iex.BadRequest, self.resource_registry_service.execute_lifecycle_transition, resource_id=rid, transition_event='NONE##') self.resource_registry_service.set_lifecycle_state( rid, LCS.INTEGRATED_PRIVATE) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.lcstate, LCS.INTEGRATED_PRIVATE) def test_association(self): # Instantiate ActorIdentity object actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create( actor_identity_obj) read_actor_identity_obj = self.resource_registry_service.read( actor_identity_obj_id) # Instantiate UserInfo object user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create( user_info_obj) read_user_info_obj = self.resource_registry_service.read( user_info_obj_id) # Test create failures with self.assertRaises(AttributeError) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.bogus, user_info_obj_id) self.assertTrue(cm.exception.message == "bogus") # Predicate not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, None, user_info_obj_id) self.assertTrue( cm.exception.message == "Association must have all elements set") # Bad association type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, 'bogustype') self.assertTrue( cm.exception.message == "Unsupported assoc_type: bogustype") # Subject id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( None, PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Association must have all elements set") # Object id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, None) self.assertTrue( cm.exception.message == "Association must have all elements set") # Bad subject id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association( "bogus", PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # Bad object id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, "bogus") self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # _id missing from subject with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj, PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Subject id or rev not available") # _id missing from object with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj) self.assertTrue( cm.exception.message == "Object id or rev not available") # Wrong subject type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( user_info_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal subject type UserInfo for predicate hasInfo") # Wrong object type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id) self.assertTrue( cm.exception.message == "Illegal object type ActorIdentity for predicate hasInfo") # Create two different association types between the same subject and predicate assoc_id1, assoc_rev1 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) # Read object, subject res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True) self.assertEquals(res_obj1, user_info_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id) self.assertEquals(res_obj2._id, actor_identity_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True) self.assertEquals(res_obj2, actor_identity_obj_id) # Create a similar association to a specific revision # TODO: This is not a supported case so far assoc_id2, assoc_rev2 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, "H2R") # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations( None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, id_only=False) ret3 = self.resource_registry_service.find_associations( predicate=PRED.hasInfo, id_only=False) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo, read_user_info_obj) ret2 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations( None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None, True) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, id_only=True) ret3 = self.resource_registry_service.find_associations( predicate=PRED.hasInfo, id_only=True) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(None, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations( actor_identity_obj_id, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations( None, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations( actor_identity_obj, None, user_info_obj_id) self.assertTrue( cm.exception.message == "Object id not available in subject") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations( actor_identity_obj_id, None, user_info_obj) self.assertTrue( cm.exception.message == "Object id not available in object") # Find subjects (good cases) subj_ret1 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) subj_ret2 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_subjects( None, PRED.hasInfo, user_info_obj_id, True) subj_ret4 = self.resource_registry_service.find_subjects( None, None, read_user_info_obj, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_subjects( None, PRED.hasInfo, user_info_obj_id, False) subj_ret6 = self.resource_registry_service.find_subjects( None, None, read_user_info_obj, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find subjects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(None, None, None) self.assertTrue(cm.exception.message == "Must provide object") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.bogus, user_info_obj_id, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_subjects( RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.hasInfo, user_info_obj, True) self.assertTrue( cm.exception.message == "Object id not available in object") # Find objects (good cases) subj_ret1 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True) subj_ret2 = self.resource_registry_service.find_objects( read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, None, True) subj_ret4 = self.resource_registry_service.find_objects( actor_identity_obj_id, None, None, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, None, False) subj_ret6 = self.resource_registry_service.find_objects( read_actor_identity_obj, None, None, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find objects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(None, None, None) self.assertTrue(cm.exception.message == "Must provide subject") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects( actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue( cm.exception.message == "Object id not available in subject") # Get association (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(None, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association( actor_identity_obj_id, None, None) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association( None, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal parameters") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association( actor_identity_obj, None, user_info_obj_id) self.assertTrue( cm.exception.message == "Object id not available in subject") with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association( actor_identity_obj_id, None, user_info_obj) self.assertTrue( cm.exception.message == "Object id not available in object") # Delete one of the associations self.resource_registry_service.delete_association(assoc_id2) assoc = self.resource_registry_service.get_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(assoc._id == assoc_id1) # Delete (bad cases) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete_association("bogus") self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # Delete other association self.resource_registry_service.delete_association(assoc_id1) # Delete resources self.resource_registry_service.delete(actor_identity_obj_id) self.resource_registry_service.delete(user_info_obj_id) def test_find_resources(self): with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_resources( RT.UserInfo, LCS.DRAFT, "name", False) self.assertTrue( cm.exception.message == "find by name does not support lcstate") ret = self.resource_registry_service.find_resources( RT.UserInfo, None, "name", False) self.assertTrue(len(ret[0]) == 0) # Instantiate an object obj = IonObject("InstrumentDevice", name="name") # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) ret = self.resource_registry_service.find_resources( RT.InstrumentDevice, None, "name", False) self.assertTrue(len(ret[0]) == 1) self.assertTrue(ret[0][0]._id == read_obj._id) ret = self.resource_registry_service.find_resources( RT.InstrumentDevice, LCS.DRAFT, None, False) self.assertTrue(len(ret[0]) == 1) self.assertTrue(ret[0][0]._id == read_obj._id) def test_attach(self): binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82" # Owner creation tests instrument = IonObject("InstrumentDevice", name='instrument') iid, _ = self.resource_registry_service.create(instrument) att = Attachment(content=binary, attachment_type=AttachmentType.BLOB) aid1 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid1) self.assertEquals(binary, att1.content) import base64 att = Attachment(content=base64.encodestring(binary), attachment_type=AttachmentType.ASCII) aid2 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid2) self.assertEquals(binary, base64.decodestring(att1.content)) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid1, aid2]) att_ids = self.resource_registry_service.find_attachments( iid, id_only=True, descending=True) self.assertEquals(att_ids, [aid2, aid1]) att_ids = self.resource_registry_service.find_attachments( iid, id_only=True, descending=True, limit=1) self.assertEquals(att_ids, [aid2]) atts = self.resource_registry_service.find_attachments(iid, id_only=False, limit=1) self.assertEquals(atts[0].content, att1.content) self.resource_registry_service.delete_attachment(aid1) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid2]) self.resource_registry_service.delete_attachment(aid2) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, []) def test_read_mult(self): test_resource1_id, _ = self.resource_registry_service.create( Resource(name='test1')) test_resource2_id, _ = self.resource_registry_service.create( Resource(name='test2')) res_list = [test_resource1_id, test_resource2_id] objects = self.resource_registry_service.read_mult(res_list) for o in objects: self.assertIsInstance(o, Resource) self.assertTrue(o._id in res_list) def test_find_associations_mult(self): dp = DataProcess() transform = Transform() pd = ProcessDefinition() dp_id, _ = self.resource_registry_service.create(dp) transform_id, _ = self.resource_registry_service.create(transform) pd_id, _ = self.resource_registry_service.create(pd) self.resource_registry_service.create_association( subject=dp_id, object=transform_id, predicate=PRED.hasTransform) self.resource_registry_service.create_association( subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition) results, _ = self.resource_registry_service.find_associations_mult( subjects=[dp_id], id_only=True) self.assertTrue(results == [transform_id]) results, _ = self.resource_registry_service.find_associations_mult( subjects=[dp_id, transform_id], id_only=True) results.sort() correct = [transform_id, pd_id] correct.sort() self.assertTrue(results == correct)
class TestResourceRegistryAttachments(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) print 'started services' def test_resource_registry_blob_sanity(self): resource_id, _ = self.RR.create(IonObject(RT.Resource, name="foo")) MY_CONTENT = "the quick brown fox etc etc etc" #save att_id = self.RR.create_attachment( resource_id, IonObject(RT.Attachment, name="test.txt", content=MY_CONTENT, content_type="text/plain", keywords=["test1", "test2"], attachment_type=AttachmentType.BLOB)) #load attachment = self.RR.read_attachment(att_id, include_content=True) self.assertEqual("test.txt", attachment.name) self.assertEqual("text/plain", attachment.content_type) self.assertIn("test1", attachment.keywords) self.assertIn("test2", attachment.keywords) #content has changed; it's base64-encoded from what we put in self.assertEqual(MY_CONTENT, attachment.content) obj = self.RR.read(resource_id) self.assertEqual(obj.name, "foo") obj.name = "TheDudeAbides" obj = self.RR.update(obj) obj = self.RR.read(resource_id) self.assertEqual(obj.name, "TheDudeAbides") att = self.RR.find_attachments(resource_id) self.assertNotEqual(att, None) actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.RR.create( actor_identity_obj) user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.RR.create(user_info_obj) assoc_id, assoc_rev = self.RR.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertNotEqual(assoc_id, None) find_assoc = self.RR.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(find_assoc[0]._id == assoc_id) subj = self.RR.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) res_obj1 = self.RR.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) self.RR.delete_association(assoc_id) self.RR.delete_attachment(att_id) self.RR.delete(resource_id)
class TestResourceRegistry(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to bank service self.resource_registry_service = ResourceRegistryServiceClient() @unittest.skip('Represents a bug in storage/retrieval') def test_tuple_in_dict(self): # create a resource with a tuple saved in a dict transform_obj = IonObject(RT.Transform) transform_obj.configuration = {} transform_obj.configuration["tuple"] = ('STRING', ) transform_id, _ = self.resource_registry_service.create(transform_obj) # read the resource back returned_transform_obj = self.resource_registry_service.read( transform_id) self.assertEqual(transform_obj.configuration["tuple"], returned_transform_obj.configuration["tuple"]) def test_basics(self): # Sequence all the tests so that we can save numerous system start and stops self._do_test_crud() self._do_test_read_mult() self._do_test_lifecycle() self._do_test_attach() self._do_test_association() self._do_test_find_resources() self._do_test_find_objects_mult() def _do_test_crud(self): # Some quick registry tests # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", name="name", foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name", "foo": "bar"}) self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name"}, foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Instantiate an object obj = IonObject("UserInfo", name="name") # Can set attributes that aren't in the object's schema with self.assertRaises(AttributeError) as cm: setattr(obj, "foo", "bar") self.assertTrue( cm.exception.message == "'UserInfo' object has no attribute 'foo'") # Cam't call update with object that hasn't been persisted with self.assertRaises(BadRequest) as cm: self.resource_registry_service.update(obj) self.assertTrue( cm.exception.message.startswith( "Object does not have required '_id' or '_rev' attribute")) # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) # Cannot create object with _id and _rev fields pre-set with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create(read_obj) # Update object read_obj.name = "John Doe" self.resource_registry_service.update(read_obj) # Update should fail with revision mismatch with self.assertRaises(Conflict) as cm: self.resource_registry_service.update(read_obj) # Re-read and update object read_obj = self.resource_registry_service.read(obj_id) self.resource_registry_service.update(read_obj) # Delete object self.resource_registry_service.delete(obj_id) # Make sure read, update and delete report error with self.assertRaises(NotFound) as cm: self.resource_registry_service.read(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.update(read_obj) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) # Owner creation tests user = IonObject("ActorIdentity", name='user') uid, _ = self.resource_registry_service.create(user) inst = IonObject("InstrumentDevice", name='instrument') iid, _ = self.resource_registry_service.create( inst, headers={'ion-actor-id': str(uid)}) ids, _ = self.resource_registry_service.find_objects(iid, PRED.hasOwner, RT.ActorIdentity, id_only=True) self.assertEquals(len(ids), 1) assoc = self.resource_registry_service.read(ids[0]) self.resource_registry_service.delete(iid) with self.assertRaises(NotFound) as ex: assoc = self.resource_registry_service.read(iid) def _do_test_read_mult(self): test_resource1_id, _ = self.resource_registry_service.create( Resource(name='test1')) test_resource2_id, _ = self.resource_registry_service.create( Resource(name='test2')) res_list = [test_resource1_id, test_resource2_id] objects = self.resource_registry_service.read_mult(res_list) for o in objects: self.assertIsInstance(o, Resource) self.assertTrue(o._id in res_list) def _do_test_lifecycle(self): # Lifecycle tests att = IonObject("InstrumentDevice", name='mine', description='desc') rid, rev = self.resource_registry_service.create(att) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.name, att.name) self.assertEquals(att1.lcstate, LCS.DRAFT) self.assertEquals(att1.availability, AS.PRIVATE) new_state = self.resource_registry_service.execute_lifecycle_transition( rid, LCE.PLAN) self.assertEquals(new_state, lcstate(LCS.PLANNED, AS.PRIVATE)) att2 = self.resource_registry_service.read(rid) self.assertEquals(att2.lcstate, LCS.PLANNED) self.assertEquals(att2.availability, AS.PRIVATE) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.execute_lifecycle_transition( rid, LCE.UNANNOUNCE) self.assertTrue( "type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce" in cm.exception.message) new_state = self.resource_registry_service.execute_lifecycle_transition( rid, LCE.DEVELOP) self.assertEquals(new_state, lcstate(LCS.DEVELOPED, AS.PRIVATE)) with self.assertRaises(BadRequest): self.resource_registry_service.execute_lifecycle_transition( resource_id=rid, transition_event='NONE##') self.resource_registry_service.set_lifecycle_state( rid, lcstate(LCS.INTEGRATED, AS.PRIVATE)) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.lcstate, LCS.INTEGRATED) self.assertEquals(att1.availability, AS.PRIVATE) def _do_test_attach(self): binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82" # Owner creation tests instrument = IonObject("InstrumentDevice", name='instrument') iid, _ = self.resource_registry_service.create(instrument) att = Attachment(content=binary, attachment_type=AttachmentType.BLOB) aid1 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment( aid1, include_content=True) self.assertEquals(binary, att1.content) import base64 att = Attachment(content=base64.encodestring(binary), attachment_type=AttachmentType.ASCII) aid2 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment( aid2, include_content=True) self.assertEquals(binary, base64.decodestring(att1.content)) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid1, aid2]) att_ids = self.resource_registry_service.find_attachments( iid, id_only=True, descending=True) self.assertEquals(att_ids, [aid2, aid1]) att_ids = self.resource_registry_service.find_attachments( iid, id_only=True, descending=True, limit=1) self.assertEquals(att_ids, [aid2]) atts = self.resource_registry_service.find_attachments( iid, id_only=False, include_content=True, limit=1) self.assertEquals(atts[0].content, binary) self.resource_registry_service.delete_attachment(aid1) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid2]) self.resource_registry_service.delete_attachment(aid2) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, []) def _do_test_association(self): # Instantiate ActorIdentity object actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create( actor_identity_obj) read_actor_identity_obj = self.resource_registry_service.read( actor_identity_obj_id) # Instantiate UserInfo object user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create( user_info_obj) read_user_info_obj = self.resource_registry_service.read( user_info_obj_id) # Test create failures with self.assertRaises(AttributeError) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.bogus, user_info_obj_id) self.assertTrue(cm.exception.message == "bogus") # Predicate not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, None, user_info_obj_id) self.assertTrue( cm.exception.message == "Association must have all elements set") # Subject id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( None, PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Association must have all elements set") # Object id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, None) self.assertTrue( cm.exception.message == "Association must have all elements set") # Bad subject id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association( "bogus", PRED.hasInfo, user_info_obj_id) self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # Bad object id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, "bogus") self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # _id missing from subject with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message.startswith("Subject id")) # _id missing from object with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj) self.assertTrue(cm.exception.message.startswith("Object id")) # Wrong subject type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( user_info_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal subject type UserInfo for predicate hasInfo") # Wrong object type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id) self.assertTrue( cm.exception.message == "Illegal object type ActorIdentity for predicate hasInfo") # Create two different association types between the same subject and predicate assoc_id1, assoc_rev1 = self.resource_registry_service.create_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) # Read object, subject res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) res_obj1 = self.resource_registry_service.read_object( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True) self.assertEquals(res_obj1, user_info_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id) self.assertEquals(res_obj2._id, actor_identity_obj_id) res_obj2 = self.resource_registry_service.read_subject( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True) self.assertEquals(res_obj2, actor_identity_obj_id) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations( None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, id_only=False) ret3 = self.resource_registry_service.find_associations( predicate=PRED.hasInfo, id_only=False) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo, read_user_info_obj) ret2 = self.resource_registry_service.find_associations( read_actor_identity_obj, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations( None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None, True) ret2 = self.resource_registry_service.find_associations( actor_identity_obj_id, PRED.hasInfo, id_only=True) ret3 = self.resource_registry_service.find_associations( predicate=PRED.hasInfo, id_only=True) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(None, None, None) self.assertIn("Illegal parameters", cm.exception.message) # Find subjects (good cases) subj_ret1 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) subj_ret2 = self.resource_registry_service.find_subjects( RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_subjects( None, PRED.hasInfo, user_info_obj_id, True) subj_ret4 = self.resource_registry_service.find_subjects( None, None, read_user_info_obj, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_subjects( None, PRED.hasInfo, user_info_obj_id, False) subj_ret6 = self.resource_registry_service.find_subjects( None, None, read_user_info_obj, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find subjects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(None, None, None) self.assertTrue(cm.exception.message == "Must provide object") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.bogus, user_info_obj_id, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_subjects( RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects( RT.UserCredentials, PRED.hasInfo, user_info_obj, True) self.assertTrue( cm.exception.message == "Object id not available in object") # Find objects (good cases) subj_ret1 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True) subj_ret2 = self.resource_registry_service.find_objects( read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, None, True) subj_ret4 = self.resource_registry_service.find_objects( actor_identity_obj_id, None, None, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, None, False) subj_ret6 = self.resource_registry_service.find_objects( read_actor_identity_obj, None, None, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find objects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(None, None, None) self.assertTrue(cm.exception.message == "Must provide subject") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_objects( actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects( actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue( cm.exception.message == "Object id not available in subject") # Get association (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(None, None, None) self.assertIn("Illegal parameters", cm.exception.message) assoc = self.resource_registry_service.get_association( actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(assoc._id == assoc_id1) # Delete (bad cases) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete_association("bogus") self.assertTrue( cm.exception.message == "Object with id bogus does not exist.") # Delete other association self.resource_registry_service.delete_association(assoc_id1) # Delete resources self.resource_registry_service.delete(actor_identity_obj_id) self.resource_registry_service.delete(user_info_obj_id) def _do_test_find_resources(self): with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_resources( RT.UserInfo, LCS.DRAFT, "name", False) self.assertTrue( cm.exception.message == "find by name does not support lcstate") ret = self.resource_registry_service.find_resources( RT.UserInfo, None, "name", False) self.assertEquals(len(ret[0]), 0) # Instantiate an object obj = IonObject("InstrumentAgentInstance", name="name") # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) ret = self.resource_registry_service.find_resources( RT.InstrumentAgentInstance, None, "name", False) self.assertEquals(len(ret[0]), 1) self.assertEquals(ret[0][0]._id, read_obj._id) ret = self.resource_registry_service.find_resources( RT.InstrumentAgentInstance, LCS.DEPLOYED, None, False) self.assertEquals(len(ret[0]), 1) self.assertEquals(ret[0][0]._id, read_obj._id) def _do_test_find_objects_mult(self): dp = DataProcess() transform = Transform() pd = ProcessDefinition() dp_id, _ = self.resource_registry_service.create(dp) transform_id, _ = self.resource_registry_service.create(transform) pd_id, _ = self.resource_registry_service.create(pd) self.resource_registry_service.create_association( subject=dp_id, object=transform_id, predicate=PRED.hasTransform) self.resource_registry_service.create_association( subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition) results, _ = self.resource_registry_service.find_objects_mult( subjects=[dp_id], id_only=True) self.assertTrue(results == [transform_id]) results, _ = self.resource_registry_service.find_objects_mult( subjects=[dp_id, transform_id], id_only=True) results.sort() correct = [transform_id, pd_id] correct.sort() self.assertTrue(results == correct) @attr('EXT') def test_get_resource_extension(self): #Testing multiple instrument owners subject1 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" actor_identity_obj1 = IonObject(RT.ActorIdentity, {"name": subject1}) actor_id1, _ = self.resource_registry_service.create( actor_identity_obj1) user_info_obj1 = IonObject(RT.UserInfo, {"name": "Foo"}) user_info_id1, _ = self.resource_registry_service.create( user_info_obj1) self.resource_registry_service.create_association( actor_id1, PRED.hasInfo, user_info_id1) subject2 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256" actor_identity_obj2 = IonObject(RT.ActorIdentity, {"name": subject2}) actor_id2, _ = self.resource_registry_service.create( actor_identity_obj2) user_info_obj2 = IonObject(RT.UserInfo, {"name": "Foo2"}) user_info_id2, _ = self.resource_registry_service.create( user_info_obj2) self.resource_registry_service.create_association( actor_id2, PRED.hasInfo, user_info_id2) test_obj = IonObject(RT.InformationResource, {"name": "TestResource"}) test_obj_id, _ = self.resource_registry_service.create(test_obj) self.resource_registry_service.create_association( test_obj_id, PRED.hasOwner, actor_id1) self.resource_registry_service.create_association( test_obj_id, PRED.hasOwner, actor_id2) extended_resource = self.resource_registry_service.get_resource_extension( test_obj_id, OT.ExtendedInformationResource) self.assertEqual(test_obj_id, extended_resource._id) self.assertEqual(len(extended_resource.owners), 2) extended_resource_list = self.resource_registry_service.get_resource_extension( str([user_info_id1, user_info_id2]), OT.ExtendedInformationResource) self.assertEqual(len(extended_resource_list), 2) optional_args = {'user_id': user_info_id1} extended_resource = self.resource_registry_service.get_resource_extension( test_obj_id, OT.TestExtendedInformationResource, optional_args=optional_args) self.assertEqual(test_obj_id, extended_resource._id) self.assertEqual(len(extended_resource.owners), 2) self.assertEqual(extended_resource.user_id, user_info_id1) @attr('PREP') def test_prepare_resource_support(self): prepare_data = self.resource_registry_service.prepare_resource_support( resource_type=RT.StreamDefinition) self.assertEqual(prepare_data.create_request.service_name, "resource_registry") self.assertEqual(prepare_data.create_request.service_operation, "create") self.assertEqual(prepare_data.create_request.request_parameters, {"object": "$(object)"}) self.assertEqual(prepare_data.update_request.service_name, "resource_registry") self.assertEqual(prepare_data.update_request.service_operation, "update") self.assertEqual(prepare_data.update_request.request_parameters, {"object": "$(object)"}) res_id, _ = self.resource_registry_service.create( prepare_data.resource) prepare_data = self.resource_registry_service.prepare_resource_support( resource_type=RT.StreamDefinition, resource_id=res_id) prepare_data.resource.name = "test_stream_def" prepare_data.resource.stream_type = "test_type" stream_def_id, _ = self.resource_registry_service.update( prepare_data.resource) #def ion_object_encoder(obj): # return obj.__dict__ #print simplejson.dumps(prepare_data, default=ion_object_encoder, indent=2) stream_def = self.resource_registry_service.read(stream_def_id) self.assertEqual(stream_def.name, prepare_data.resource.name) self.assertEqual(stream_def.stream_type, prepare_data.resource.stream_type)
class TestTransformWorker(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Instantiate a process to represent the test process=TransformWorkerTestProcess() self.dataset_management_client = DatasetManagementServiceClient(node=self.container.node) self.pubsub_client = PubsubManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process = process) self.time_dom, self.spatial_dom = time_series_domain() self.ph = ParameterHelper(self.dataset_management_client, self.addCleanup) self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10) def push_granule(self, data_product_id): ''' Publishes and monitors that the granule arrived ''' datasets, _ = self.rrclient.find_objects(data_product_id, PRED.hasDataset, id_only=True) dataset_monitor = DatasetMonitor(datasets[0]) rdt = self.ph.rdt_for_data_product(data_product_id) self.ph.fill_parsed_rdt(rdt) self.ph.publish_rdt_to_data_product(data_product_id, rdt) assert dataset_monitor.wait() dataset_monitor.stop() @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_transform_worker(self): # test that a data process (type: data-product-in / data-product-out) can be defined and launched. # verify that the output granule fields are correctly populated # test that the input and output data products are linked to facilitate provenance self.dp_list = [] self.data_process_objs = [] self._output_stream_ids = [] self.granule_verified = Event() self.worker_assigned_event_verified = Event() self.dp_created_event_verified = Event() self.heartbeat_event_verified = Event() self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True) # create the StreamDefinition self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id) self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id) # create the DataProduct that is the input to the data processes input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream', temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump()) self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # retrieve the Stream for this data product stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True) self.stream_id = stream_ids[0] self.start_event_listener() # create the DPD, DataProcess and output DataProduct dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process() self.dp_list.append(dataprocess_id) # validate the repository for data product algorithms persists the new resources NEW SA-1 # create_data_process call created one of each dpd_ids, _ = self.rrclient.find_resources(restype=OT.DataProcessDefinition, id_only=False) # there will be more than one becuase of the DPDs that reperesent the PFs in the data product above self.assertTrue(dpd_ids is not None) dp_ids, _ = self.rrclient.find_resources(restype=OT.DataProcess, id_only=False) # only one DP becuase the PFs that are in the code dataproduct above are not activated yet. self.assertEquals(len(dp_ids), 1) # validate the name and version label NEW SA - 2 dataprocessdef_obj = self.dataprocessclient.read_data_process_definition(dataprocessdef_id) self.assertEqual(dataprocessdef_obj.version_label, '1.0a') self.assertEqual(dataprocessdef_obj.name, 'add_arrays') # validate that the DPD has an attachment NEW SA - 21 attachment_ids, assoc_ids = self.rrclient.find_objects(dataprocessdef_id, PRED.hasAttachment, RT.Attachment, True) self.assertEqual(len(attachment_ids), 1) attachment_obj = self.rrclient.read_attachment(attachment_ids[0]) log.debug('attachment: %s', attachment_obj) # validate that the data process resource has input and output data products associated # L4-CI-SA-RQ-364 and NEW SA-3 outproduct_ids, assoc_ids = self.rrclient.find_objects(dataprocess_id, PRED.hasOutputProduct, RT.DataProduct, True) self.assertEqual(len(outproduct_ids), 1) inproduct_ids, assoc_ids = self.rrclient.find_objects(dataprocess_id, PRED.hasInputProduct, RT.DataProduct, True) self.assertEqual(len(inproduct_ids), 1) # Test for provenance. Get Data product produced by the data processes output_data_product_id,_ = self.rrclient.find_objects(subject=dataprocess_id, object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, id_only=True) output_data_product_provenance = self.dataproductclient.get_data_product_provenance(output_data_product_id[0]) # Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the # DataProcessDefinition creating the child from the parent. self.assertTrue(len(output_data_product_provenance) == 2) self.assertTrue(self.input_dp_id in output_data_product_provenance[output_data_product_id[0]]['parents']) self.assertTrue(output_data_product_provenance[output_data_product_id[0]]['parents'][self.input_dp_id]['data_process_definition_id'] == dataprocessdef_id) # NEW SA - 4 | Data processing shall include the appropriate data product algorithm name and version number in # the metadata of each output data product created by the data product algorithm. output_data_product_obj,_ = self.rrclient.find_objects(subject=dataprocess_id, object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, id_only=False) self.assertTrue(output_data_product_obj[0].name != None) self.assertTrue(output_data_product_obj[0]._rev != None) # retrieve subscription from data process subscription_objs, _ = self.rrclient.find_objects(subject=dataprocess_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False) log.debug('test_transform_worker subscription_obj: %s', subscription_objs[0]) # create a queue to catch the published granules self.subscription_id = self.pubsub_client.create_subscription(name='parsed_subscription', stream_ids=[self.stream_id], exchange_name=subscription_objs[0].exchange_name) self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id) self.pubsub_client.activate_subscription(self.subscription_id) self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id) stream_route = self.pubsub_client.read_stream_route(self.stream_id) self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id, stream_route=stream_route ) for n in range(1, 101): rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) rdt['time'] = [0] # time should always come first rdt['conductivity'] = [1] rdt['pressure'] = [2] rdt['salinity'] = [8] self.publisher.publish(rdt.to_granule()) # validate that the output granule is received and the updated value is correct self.assertTrue(self.granule_verified.wait(self.wait_time)) # validate that the data process loaded into worker event is received (L4-CI-SA-RQ-182) self.assertTrue(self.worker_assigned_event_verified.wait(self.wait_time)) # validate that the data process create (with data product ids) event is received (NEW SA -42) self.assertTrue(self.dp_created_event_verified.wait(self.wait_time)) # validate that the data process heartbeat event is received (for every hundred granules processed) (L4-CI-SA-RQ-182) #this takes a while so set wait limit to large value self.assertTrue(self.heartbeat_event_verified.wait(200)) # validate that the code from the transform function can be retrieve via inspect_data_process_definition src = self.dataprocessclient.inspect_data_process_definition(dataprocessdef_id) self.assertIn( 'def add_arrays(a, b)', src) # now delete the DPD and DP then verify that the resources are retired so that information required for provenance are still available self.dataprocessclient.delete_data_process(dataprocess_id) self.dataprocessclient.delete_data_process_definition(dataprocessdef_id) in_dp_objs, _ = self.rrclient.find_objects(subject=dataprocess_id, predicate=PRED.hasInputProduct, object_type=RT.DataProduct, id_only=True) self.assertTrue(in_dp_objs is not None) dpd_objs, _ = self.rrclient.find_subjects(subject_type=RT.DataProcessDefinition, predicate=PRED.hasDataProcess, object=dataprocess_id, id_only=True) self.assertTrue(dpd_objs is not None) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_transform_worker_with_instrumentdevice(self): # test that a data process (type: data-product-in / data-product-out) can be defined and launched. # verify that the output granule fields are correctly populated # test that the input and output data products are linked to facilitate provenance self.data_process_objs = [] self._output_stream_ids = [] self.event_verified = Event() # Create CTD Parsed as the initial data product # create a stream definition for the data from the ctd simulator self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id) # create the DataProduct that is the input to the data processes input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream', temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump()) self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # retrieve the Stream for this data product stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True) self.stream_id = stream_ids[0] log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id) # only ever need one device for testing purposes. instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='test_ctd_device') if instDevice_obj: instDevice_id = instDevice_obj[0]._id else: instDevice_obj = IonObject(RT.InstrumentDevice, name='test_ctd_device', description="test_ctd_device", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=self.input_dp_id) # create the DPD, DataProcess and output DataProduct dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process() self.addCleanup(self.dataprocessclient.delete_data_process, dataprocess_id) self.addCleanup(self.dataprocessclient.delete_data_process_definition, dataprocessdef_id) # Test for provenance. Get Data product produced by the data processes output_data_product_id,_ = self.rrclient.find_objects(subject=dataprocess_id, object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, id_only=True) output_data_product_provenance = self.dataproductclient.get_data_product_provenance(output_data_product_id[0]) # Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the # DataProcessDefinition creating the child from the parent. self.assertTrue(len(output_data_product_provenance) == 3) self.assertTrue(self.input_dp_id in output_data_product_provenance[output_data_product_id[0]]['parents']) self.assertTrue(instDevice_id in output_data_product_provenance[self.input_dp_id]['parents']) self.assertTrue(output_data_product_provenance[instDevice_id]['type'] == 'InstrumentDevice') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_transform_worker_with_platformdevice(self): # test that a data process (type: data-product-in / data-product-out) can be defined and launched. # verify that the output granule fields are correctly populated # test that the input and output data products are linked to facilitate provenance self.data_process_objs = [] self._output_stream_ids = [] self.event_verified = Event() # Create CTD Parsed as the initial data product # create a stream definition for the data from the ctd simulator self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id) # create the DataProduct that is the input to the data processes input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream', temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump()) self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # retrieve the Stream for this data product stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True) self.stream_id = stream_ids[0] log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id) # only ever need one device for testing purposes. platform_device_obj,_ = self.rrclient.find_resources(restype=RT.PlatformDevice, name='TestPlatform') if platform_device_obj: platform_device_id = platform_device_obj[0]._id else: platform_device_obj = IonObject(RT.PlatformDevice, name='TestPlatform', description="TestPlatform", serial_number="12345" ) platform_device_id = self.imsclient.create_platform_device(platform_device=platform_device_obj) self.damsclient.assign_data_product(input_resource_id=platform_device_id, data_product_id=self.input_dp_id) # create the DPD, DataProcess and output DataProduct dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process() self.addCleanup(self.dataprocessclient.delete_data_process, dataprocess_id) self.addCleanup(self.dataprocessclient.delete_data_process_definition, dataprocessdef_id) # Test for provenance. Get Data product produced by the data processes output_data_product_id,_ = self.rrclient.find_objects(subject=dataprocess_id, object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, id_only=True) output_data_product_provenance = self.dataproductclient.get_data_product_provenance(output_data_product_id[0]) # Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the # DataProcessDefinition creating the child from the parent. self.assertTrue(len(output_data_product_provenance) == 3) self.assertTrue(self.input_dp_id in output_data_product_provenance[output_data_product_id[0]]['parents']) self.assertTrue(platform_device_id in output_data_product_provenance[self.input_dp_id]['parents']) self.assertTrue(output_data_product_provenance[platform_device_id]['type'] == 'PlatformDevice') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_event_transform_worker(self): self.data_process_objs = [] self._output_stream_ids = [] self.event_verified = Event() # test that a data process (type: data-product-in / event-out) can be defined and launched. # verify that event fields are correctly populated self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True) # create the StreamDefinition self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id) self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id) # create the DataProduct input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream', temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump()) self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # retrieve the Stream for this data product stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True) self.stream_id = stream_ids[0] # create the DPD and two DPs self.event_data_process_id = self.create_event_data_processes() # retrieve subscription from data process subscription_objs, _ = self.rrclient.find_objects(subject=self.event_data_process_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False) log.debug('test_event_transform_worker subscription_obj: %s', subscription_objs[0]) # create a queue to catch the published granules self.subscription_id = self.pubsub_client.create_subscription(name='parsed_subscription', stream_ids=[self.stream_id], exchange_name=subscription_objs[0].exchange_name) self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id) self.pubsub_client.activate_subscription(self.subscription_id) self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id) stream_route = self.pubsub_client.read_stream_route(self.stream_id) self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id, stream_route=stream_route ) self.start_event_transform_listener() self.data_modified = Event() rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) rdt['time'] = [0] # time should always come first rdt['conductivity'] = [1] rdt['pressure'] = [2] rdt['salinity'] = [8] self.publisher.publish(rdt.to_granule()) self.assertTrue(self.event_verified.wait(self.wait_time)) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_bad_argument_map(self): self._output_stream_ids = [] # test that a data process (type: data-product-in / data-product-out) parameter mapping it validated during # data process creation and that the correct exception is raised for both input and output. self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True) # create the StreamDefinition self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id) self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id) # create the DataProduct that is the input to the data processes input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream', temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump()) self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id) # two data processes using one transform and one DPD dp1_func_output_dp_id = self.create_output_data_product() # Set up DPD and DP #2 - array add function tf_obj = IonObject(RT.TransformFunction, name='add_array_func', description='adds values in an array', function='add_arrays', module="ion_example.add_arrays", arguments=['arr1', 'arr2'], function_type=TransformFunctionType.TRANSFORM, uri='http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg' ) add_array_func_id, rev = self.rrclient.create(tf_obj) dpd_obj = IonObject(RT.DataProcessDefinition, name='add_arrays', description='adds the values of two arrays', data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS ) add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='add_array_func' ) # create the data process with invalid argument map argument_map = {"arr1": "foo", "arr2": "bar"} output_param = "salinity" with self.assertRaises(BadRequest) as cm: dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id], outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param) ex = cm.exception log.debug(' exception raised: %s', cm) self.assertEqual(ex.message, "Input data product does not contain the parameters defined in argument map") # create the data process with invalid output parameter name argument_map = {"arr1": "conductivity", "arr2": "pressure"} output_param = "foo" with self.assertRaises(BadRequest) as cm: dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id], outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param) ex = cm.exception log.debug(' exception raised: %s', cm) self.assertEqual(ex.message, "Output data product does not contain the output parameter name provided") def create_event_data_processes(self): # two data processes using one transform and one DPD argument_map= {"a": "salinity"} # set up DPD and DP #2 - array add function tf_obj = IonObject(RT.TransformFunction, name='validate_salinity_array', description='validate_salinity_array', function='validate_salinity_array', module="ion.processes.data.transforms.test.test_transform_worker", arguments=['a'], function_type=TransformFunctionType.TRANSFORM ) add_array_func_id, rev = self.rrclient.create(tf_obj) dpd_obj = IonObject(RT.DataProcessDefinition, name='validate_salinity_array', description='validate_salinity_array', data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS, ) add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='validate_salinity_array' ) # create the data process dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id], outputs=None, argument_map=argument_map) self.damsclient.register_process(dp1_data_process_id) self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id) return dp1_data_process_id def create_data_process(self): # two data processes using one transform and one DPD dp1_func_output_dp_id = self.create_output_data_product() argument_map = {"arr1": "conductivity", "arr2": "pressure"} output_param = "salinity" # set up DPD and DP #2 - array add function tf_obj = IonObject(RT.TransformFunction, name='add_array_func', description='adds values in an array', function='add_arrays', module="ion_example.add_arrays", arguments=['arr1', 'arr2'], function_type=TransformFunctionType.TRANSFORM, uri='http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg' ) add_array_func_id, rev = self.rrclient.create(tf_obj) dpd_obj = IonObject(RT.DataProcessDefinition, name='add_arrays', description='adds the values of two arrays', data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS, version_label='1.0a' ) add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='add_array_func' ) # create the data process dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id], outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param) self.damsclient.register_process(dp1_data_process_id) #self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id) # add an attachment object to this DPD to test new SA-21 import msgpack attachment_content = 'foo bar' attachment_obj = IonObject( RT.Attachment, name='test_attachment', attachment_type=AttachmentType.ASCII, content_type='text/plain', content=msgpack.packb(attachment_content)) att_id = self.rrclient.create_attachment(add_array_dpd_id, attachment_obj) self.addCleanup(self.rrclient.delete_attachment, att_id) return add_array_dpd_id, dp1_data_process_id, dp1_func_output_dp_id def create_output_data_product(self): dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id) dp1_output_dp_obj = IonObject( RT.DataProduct, name='data_process1_data_product', description='output of add array func', temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump()) dp1_func_output_dp_id = self.dataproductclient.create_data_product(dp1_output_dp_obj, dp1_outgoing_stream_id) self.addCleanup(self.dataproductclient.delete_data_product, dp1_func_output_dp_id) # retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id, PRED.hasStream, None, True) self._output_stream_ids.append(stream_ids[0]) subscription_id = self.pubsub_client.create_subscription('validator', data_product_ids=[dp1_func_output_dp_id]) self.addCleanup(self.pubsub_client.delete_subscription, subscription_id) def on_granule(msg, route, stream_id): log.debug('recv_packet stream_id: %s route: %s msg: %s', stream_id, route, msg) self.validate_output_granule(msg, route, stream_id) self.granule_verified.set() validator = StandaloneStreamSubscriber('validator', callback=on_granule) validator.start() self.addCleanup(validator.stop) self.pubsub_client.activate_subscription(subscription_id) self.addCleanup(self.pubsub_client.deactivate_subscription, subscription_id) return dp1_func_output_dp_id def validate_event(self, *args, **kwargs): """ This method is a callback function for receiving DataProcessStatusEvent. """ data_process_event = args[0] log.debug("DataProcessStatusEvent: %s" , str(data_process_event.__dict__)) # if data process already created, check origin if self.dp_list: self.assertIn( data_process_event.origin, self.dp_list) # if this is a heartbeat event then 100 granules have been processed if 'data process status update.' in data_process_event.description: self.heartbeat_event_verified.set() else: # else check that this is the assign event if 'Data process assigned to transform worker' in data_process_event.description: self.worker_assigned_event_verified.set() elif 'Data process created for data product' in data_process_event.description: self.dp_created_event_verified.set() def validate_output_granule(self, msg, route, stream_id): self.assertIn( stream_id, self._output_stream_ids) rdt = RecordDictionaryTool.load_from_granule(msg) log.debug('validate_output_granule rdt: %s', rdt) sal_val = rdt['salinity'] np.testing.assert_array_equal(sal_val, np.array([3])) def start_event_listener(self): es = EventSubscriber(event_type=OT.DataProcessStatusEvent, callback=self.validate_event) es.start() self.addCleanup(es.stop) def validate_transform_event(self, *args, **kwargs): """ This method is a callback function for receiving DataProcessStatusEvent. """ status_alert_event = args[0] np.testing.assert_array_equal(status_alert_event.origin, self.stream_id ) np.testing.assert_array_equal(status_alert_event.values, np.array([self.event_data_process_id])) log.debug("DeviceStatusAlertEvent: %s" , str(status_alert_event.__dict__)) self.event_verified.set() def start_event_transform_listener(self): es = EventSubscriber(event_type=OT.DeviceStatusAlertEvent, callback=self.validate_transform_event) es.start() self.addCleanup(es.stop) def test_download(self): egg_url = 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg' egg_path = TransformWorker.download_egg(egg_url) import pkg_resources pkg_resources.working_set.add_entry(egg_path) from ion_example.add_arrays import add_arrays a = add_arrays(1,2) self.assertEquals(a,3)
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.dataset_management = self.datasetclient self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentModel id = %s', instModel_id) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD) try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") try: instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s', instDevice_id) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance") self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) log.info( 'TestDataProcessWithLookupTable: new Stream Definition id = %s', instDevice_id) log.info('Creating new CDM data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product( dp_obj, ctd_stream_def_id) log.info('new ctd_parsed_data_product_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.info('TestDataProcessWithLookupTable: Data product streams1 = %s', stream_ids) #------------------------------- # Create CTD Raw as the second data product #------------------------------- log.info( 'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition' ) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='ctd_raw', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) ctd_raw_data_product = self.dataproductclient.create_data_product( dp_obj, raw_stream_def_id) log.info('new ctd_raw_data_product_id = %s', ctd_raw_data_product) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.info('Data product streams2 = %s', stream_ids) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug( "TestDataProcessWithLookupTable: create data process definition ctd_L0_all" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new ctd_L0_all data process definition: %s" % ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment( ctd_L0_all_dprocdef_id, att) log.debug( "TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment)) processDefinitionAttachment_obj = self.rrclient.read( processDefinitionAttachment) log.debug( "TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj)) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity') outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure') outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature') log.debug( "TestDataProcessWithLookupTable: create output data product L0 conductivity" ) ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug( "TestDataProcessWithLookupTable: create output data product L0 pressure" ) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug( "TestDataProcessWithLookupTable: create output data product L0 temperature" ) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug( "TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) out_prods = [ ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id ] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=ctd_L0_all_dprocdef_id, in_data_product_ids=in_prods, out_data_product_ids=out_prods) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "TestDataProcessWithLookupTable: create L0 all data_process return" ) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, object_type=RT.Process, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment( ctd_l0_all_data_process_id, att) log.info( 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s', processAttachment)
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) def test_get_data_product_provenance_report(self): #Create a test device device_obj = Device(name='Device1', description='test instrument site') device_id, _ = self.rrclient.create(device_obj) self.addCleanup(self.rrclient.delete, device_id) #Create a test DataProduct data_product1_obj = DataProduct(name='DataProduct1', description='test data product 1') data_product1_id, _ = self.rrclient.create(data_product1_obj) self.addCleanup(self.rrclient.delete, data_product1_id) #Create a test DataProcess data_process_obj = DataProcess(name='DataProcess', description='test data process') data_process_id, _ = self.rrclient.create(data_process_obj) self.addCleanup(self.rrclient.delete, data_process_id) #Create a second test DataProduct data_product2_obj = DataProduct(name='DataProduct2', description='test data product 2') data_product2_id, _ = self.rrclient.create(data_product2_obj) self.addCleanup(self.rrclient.delete, data_product2_id) #Create a test DataProducer data_producer_obj = DataProducer(name='DataProducer', description='test data producer') data_producer_id, rev = self.rrclient.create(data_producer_obj) #Link the DataProcess to the second DataProduct manually assoc_id, _ = self.rrclient.create_association(subject=data_process_id, predicate=PRED.hasInputProduct, object=data_product2_id) self.addCleanup(self.rrclient.delete_association, assoc_id) # Register the instrument and process. This links the device and the data process # with their own producers self.damsclient.register_instrument(device_id) self.addCleanup(self.damsclient.unregister_instrument, device_id) self.damsclient.register_process(data_process_id) self.addCleanup(self.damsclient.unregister_process, data_process_id) #Manually link the first DataProduct with the test DataProducer assoc_id, _ = self.rrclient.create_association(subject=data_product1_id, predicate=PRED.hasDataProducer, object=data_producer_id) #Get the DataProducer linked to the DataProcess (created in register_process above) #Associate that with with DataProduct1's DataProducer data_process_producer_ids, _ = self.rrclient.find_objects(subject=data_process_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True) assoc_id, _ = self.rrclient.create_association(subject=data_process_producer_ids[0], predicate=PRED.hasParent, object=data_producer_id) self.addCleanup(self.rrclient.delete_association, assoc_id) #Get the DataProducer linked to the Device (created in register_instrument #Associate that with the DataProcess's DataProducer device_producer_ids, _ = self.rrclient.find_objects(subject=device_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True) assoc_id, _ = self.rrclient.create_association(subject=data_producer_id, predicate=PRED.hasParent, object=device_producer_ids[0]) #Create the links between the Device, DataProducts, DataProcess, and all DataProducers self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=data_product1_id) self.addCleanup(self.damsclient.unassign_data_product, device_id, data_product1_id) self.damsclient.assign_data_product(input_resource_id=data_process_id, data_product_id=data_product2_id) self.addCleanup(self.damsclient.unassign_data_product, data_process_id, data_product2_id) #Traverse through the relationships to get the links between objects res = self.dpmsclient.get_data_product_provenance_report(data_product2_id) #Make sure there are four keys self.assertEqual(len(res.keys()), 4) parent_count = 0 config_count = 0 for v in res.itervalues(): if 'parent' in v: parent_count += 1 if 'config' in v: config_count += 1 #Make sure there are three parents and four configs self.assertEqual(parent_count, 3) self.assertEqual(config_count, 4) @unittest.skip('This test is obsolete with new framework') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict' ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [parsed_config] ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: input_data_products = [ctd_parsed_data_product] output_data_products = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L0_all_dprocdef_id, in_data_product_ids = input_data_products, out_data_product_ids = output_data_products ) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_conductivity_dprocdef_id, in_data_product_ids = [ctd_l0_conductivity_output_dp_id], out_data_product_ids = [ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_pressure_dprocdef_id, in_data_product_ids = [ctd_l0_pressure_output_dp_id], out_data_product_ids = [ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_temperature_dprocdef_id, in_data_product_ids = [ctd_l0_temperature_output_dp_id], out_data_product_ids = [ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L2_salinity_dprocdef_id, in_data_product_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], out_data_product_ids = [ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: in_dp_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id] out_dp_ids = [ctd_l2_density_output_dp_id] l2_density_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L2_density_dprocdef_id, in_data_product_ids = in_dp_ids, out_data_product_ids = out_dp_ids) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) self.assertEqual(1, len(extended_product.data_processes) ) self.assertEqual(3, len(extended_product.process_input_data_products) ) # log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) # log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id) print results #------------------------------- # Cleanup #------------------------------- self.dpmsclient.delete_data_product(ctd_parsed_data_product) self.dpmsclient.delete_data_product(log_data_product_id) self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestResourceRegistry(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to bank service self.resource_registry_service = ResourceRegistryServiceClient() @unittest.skip('Represents a bug in storage/retrieval') def test_tuple_in_dict(self): # create a resource with a tuple saved in a dict transform_obj = IonObject(RT.Transform) transform_obj.configuration = {} transform_obj.configuration["tuple"] = ('STRING',) transform_id, _ = self.resource_registry_service.create(transform_obj) # read the resource back returned_transform_obj = self.resource_registry_service.read(transform_id) self.assertEqual(transform_obj.configuration["tuple"], returned_transform_obj.configuration["tuple"]) def test_basics(self): # Sequence all the tests so that we can save numerous system start and stops self._do_test_crud() self._do_test_read_mult() self._do_test_lifecycle() self._do_test_attach() self._do_test_association() self._do_test_find_resources() self._do_test_find_objects_mult() def _do_test_crud(self): # Some quick registry tests # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", name="name", foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name", "foo": "bar"}) self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Can't call new with fields that aren't defined in the object's schema with self.assertRaises(TypeError) as cm: IonObject("UserInfo", {"name": "name"}, foo="bar") self.assertTrue(cm.exception.message == "__init__() got an unexpected keyword argument 'foo'") # Instantiate an object obj = IonObject("UserInfo", name="name") # Can set attributes that aren't in the object's schema with self.assertRaises(AttributeError) as cm: setattr(obj, "foo", "bar") self.assertTrue(cm.exception.message == "'UserInfo' object has no attribute 'foo'") # Cam't call update with object that hasn't been persisted with self.assertRaises(BadRequest) as cm: self.resource_registry_service.update(obj) self.assertTrue(cm.exception.message.startswith("Object does not have required '_id' or '_rev' attribute")) # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) # Cannot create object with _id and _rev fields pre-set with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create(read_obj) # Update object read_obj.name = "John Doe" self.resource_registry_service.update(read_obj) # Update should fail with revision mismatch with self.assertRaises(Conflict) as cm: self.resource_registry_service.update(read_obj) # Re-read and update object read_obj = self.resource_registry_service.read(obj_id) self.resource_registry_service.update(read_obj) # Delete object self.resource_registry_service.delete(obj_id) # Make sure read, update and delete report error with self.assertRaises(NotFound) as cm: self.resource_registry_service.read(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.update(read_obj) self.assertTrue(cm.exception.message.startswith("Object with id")) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete(obj_id) self.assertTrue(cm.exception.message.startswith("Object with id")) # Owner creation tests user = IonObject("ActorIdentity", name='user') uid,_ = self.resource_registry_service.create(user) inst = IonObject("InstrumentDevice", name='instrument') iid,_ = self.resource_registry_service.create(inst, headers={'ion-actor-id':str(uid)}) ids,_ = self.resource_registry_service.find_objects(iid, PRED.hasOwner, RT.ActorIdentity, id_only=True) self.assertEquals(len(ids), 1) assoc = self.resource_registry_service.read(ids[0]) self.resource_registry_service.delete(iid) with self.assertRaises(NotFound) as ex: assoc = self.resource_registry_service.read(iid) def _do_test_read_mult(self): test_resource1_id,_ = self.resource_registry_service.create(Resource(name='test1')) test_resource2_id,_ = self.resource_registry_service.create(Resource(name='test2')) res_list = [test_resource1_id, test_resource2_id] objects = self.resource_registry_service.read_mult(res_list) for o in objects: self.assertIsInstance(o,Resource) self.assertTrue(o._id in res_list) def _do_test_lifecycle(self): # Lifecycle tests att = IonObject("InstrumentDevice", name='mine', description='desc') rid,rev = self.resource_registry_service.create(att) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.name, att.name) self.assertEquals(att1.lcstate, LCS.DRAFT) self.assertEquals(att1.availability, AS.PRIVATE) new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.PLAN) self.assertEquals(new_state, lcstate(LCS.PLANNED, AS.PRIVATE)) att2 = self.resource_registry_service.read(rid) self.assertEquals(att2.lcstate, LCS.PLANNED) self.assertEquals(att2.availability, AS.PRIVATE) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.execute_lifecycle_transition(rid, LCE.UNANNOUNCE) self.assertTrue("type=InstrumentDevice, lcstate=PLANNED_PRIVATE has no transition for event unannounce" in cm.exception.message) new_state = self.resource_registry_service.execute_lifecycle_transition(rid, LCE.DEVELOP) self.assertEquals(new_state, lcstate(LCS.DEVELOPED, AS.PRIVATE)) with self.assertRaises(BadRequest): self.resource_registry_service.execute_lifecycle_transition( resource_id=rid, transition_event='NONE##') self.resource_registry_service.set_lifecycle_state(rid, lcstate(LCS.INTEGRATED, AS.PRIVATE)) att1 = self.resource_registry_service.read(rid) self.assertEquals(att1.lcstate, LCS.INTEGRATED) self.assertEquals(att1.availability, AS.PRIVATE) def _do_test_attach(self): binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82" # Owner creation tests instrument = IonObject("InstrumentDevice", name='instrument') iid,_ = self.resource_registry_service.create(instrument) att = Attachment(content=binary, attachment_type=AttachmentType.BLOB) aid1 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid1, include_content=True) self.assertEquals(binary, att1.content) import base64 att = Attachment(content=base64.encodestring(binary), attachment_type=AttachmentType.ASCII) aid2 = self.resource_registry_service.create_attachment(iid, att) att1 = self.resource_registry_service.read_attachment(aid2, include_content=True) self.assertEquals(binary, base64.decodestring(att1.content)) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid1, aid2]) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True) self.assertEquals(att_ids, [aid2, aid1]) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True, descending=True, limit=1) self.assertEquals(att_ids, [aid2]) atts = self.resource_registry_service.find_attachments(iid, id_only=False, include_content=True, limit=1) self.assertEquals(atts[0].content, binary) self.resource_registry_service.delete_attachment(aid1) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, [aid2]) self.resource_registry_service.delete_attachment(aid2) att_ids = self.resource_registry_service.find_attachments(iid, id_only=True) self.assertEquals(att_ids, []) def _do_test_association(self): # Instantiate ActorIdentity object actor_identity_obj = IonObject("ActorIdentity", name="name") actor_identity_obj_id, actor_identity_obj_rev = self.resource_registry_service.create(actor_identity_obj) read_actor_identity_obj = self.resource_registry_service.read(actor_identity_obj_id) # Instantiate UserInfo object user_info_obj = IonObject("UserInfo", name="name") user_info_obj_id, user_info_obj_rev = self.resource_registry_service.create(user_info_obj) read_user_info_obj = self.resource_registry_service.read(user_info_obj_id) # Test create failures with self.assertRaises(AttributeError) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.bogus, user_info_obj_id) self.assertTrue(cm.exception.message == "bogus") # Predicate not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, None, user_info_obj_id) self.assertTrue(cm.exception.message == "Association must have all elements set") # Subject id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(None, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Association must have all elements set") # Object id or object not provided with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, None) self.assertTrue(cm.exception.message == "Association must have all elements set") # Bad subject id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association("bogus", PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Object with id bogus does not exist.") # Bad object id with self.assertRaises(NotFound) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, "bogus") self.assertTrue(cm.exception.message == "Object with id bogus does not exist.") # _id missing from subject with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message.startswith("Subject id")) # _id missing from object with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj) self.assertTrue(cm.exception.message.startswith("Object id")) # Wrong subject type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(user_info_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(cm.exception.message == "Illegal subject type UserInfo for predicate hasInfo") # Wrong object type with self.assertRaises(BadRequest) as cm: self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, actor_identity_obj_id) self.assertTrue(cm.exception.message == "Illegal object type ActorIdentity for predicate hasInfo") # Create two different association types between the same subject and predicate assoc_id1, assoc_rev1 = self.resource_registry_service.create_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) # Read object, subject res_obj1 = self.resource_registry_service.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo) self.assertEquals(res_obj1._id, user_info_obj_id) res_obj1 = self.resource_registry_service.read_object(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, id_only=True) self.assertEquals(res_obj1, user_info_obj_id) res_obj2 = self.resource_registry_service.read_subject(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id) self.assertEquals(res_obj2._id, actor_identity_obj_id) res_obj2 = self.resource_registry_service.read_subject(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, id_only=True) self.assertEquals(res_obj2, actor_identity_obj_id) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id, None, False) ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=False) ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=False) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (good cases) ret1 = self.resource_registry_service.find_associations(read_actor_identity_obj, PRED.hasInfo, read_user_info_obj) ret2 = self.resource_registry_service.find_associations(read_actor_identity_obj, PRED.hasInfo) ret3 = self.resource_registry_service.find_associations(None, PRED.hasInfo) self.assertTrue(len(ret1) == len(ret2) == len(ret3)) self.assertTrue(ret1[0]._id == ret2[0]._id == ret3[0]._id) ret1 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, read_user_info_obj, None, True) ret2 = self.resource_registry_service.find_associations(actor_identity_obj_id, PRED.hasInfo, id_only=True) ret3 = self.resource_registry_service.find_associations(predicate=PRED.hasInfo, id_only=True) self.assertTrue(ret1 == ret2 == ret3) # Search for associations (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_associations(None, None, None) self.assertIn("Illegal parameters", cm.exception.message) # Find subjects (good cases) subj_ret1 = self.resource_registry_service.find_subjects(RT.ActorIdentity, PRED.hasInfo, user_info_obj_id, True) subj_ret2 = self.resource_registry_service.find_subjects(RT.ActorIdentity, PRED.hasInfo, read_user_info_obj, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, True) subj_ret4 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_subjects(None, PRED.hasInfo, user_info_obj_id, False) subj_ret6 = self.resource_registry_service.find_subjects(None, None, read_user_info_obj, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find subjects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(None, None, None) self.assertTrue(cm.exception.message == "Must provide object") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.bogus, user_info_obj_id, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_subjects(RT.UserInfo, PRED.hasCredentials, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj_id, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_subjects(RT.UserCredentials, PRED.hasInfo, user_info_obj, True) self.assertTrue(cm.exception.message == "Object id not available in object") # Find objects (good cases) subj_ret1 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserInfo, True) subj_ret2 = self.resource_registry_service.find_objects(read_actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue(len(subj_ret1) == len(subj_ret2)) self.assertTrue(subj_ret1[0] == subj_ret2[0]) self.assertTrue(subj_ret1[1][0]._id == subj_ret2[1][0]._id) subj_ret3 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, True) subj_ret4 = self.resource_registry_service.find_objects(actor_identity_obj_id, None, None, True) self.assertTrue(len(subj_ret3) == len(subj_ret4)) self.assertTrue(subj_ret3[0] == subj_ret4[0]) self.assertTrue(subj_ret3[1][0]._id == subj_ret4[1][0]._id) subj_ret5 = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, None, False) subj_ret6 = self.resource_registry_service.find_objects(read_actor_identity_obj, None, None, False) self.assertTrue(len(subj_ret5) == len(subj_ret6)) self.assertTrue(subj_ret5[0][0]._id == subj_ret6[0][0]._id) self.assertTrue(subj_ret5[1][0]._id == subj_ret6[1][0]._id) # Find objects (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(None, None, None) self.assertTrue(cm.exception.message == "Must provide subject") with self.assertRaises(AttributeError) as cm: self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.bogus, RT.UserCredentials, True) self.assertTrue(cm.exception.message == "bogus") ret = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasCredentials, RT.ActorIdentity, True) self.assertTrue(len(ret[0]) == 0) ret = self.resource_registry_service.find_objects(actor_identity_obj_id, PRED.hasInfo, RT.UserCredentials, True) self.assertTrue(len(ret[0]) == 0) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_objects(actor_identity_obj, PRED.hasInfo, RT.UserInfo, True) self.assertTrue(cm.exception.message == "Object id not available in subject") # Get association (bad cases) with self.assertRaises(BadRequest) as cm: self.resource_registry_service.get_association(None, None, None) self.assertIn("Illegal parameters", cm.exception.message) assoc = self.resource_registry_service.get_association(actor_identity_obj_id, PRED.hasInfo, user_info_obj_id) self.assertTrue(assoc._id == assoc_id1) # Delete (bad cases) with self.assertRaises(NotFound) as cm: self.resource_registry_service.delete_association("bogus") self.assertTrue(cm.exception.message == "Object with id bogus does not exist.") # Delete other association self.resource_registry_service.delete_association(assoc_id1) # Delete resources self.resource_registry_service.delete(actor_identity_obj_id) self.resource_registry_service.delete(user_info_obj_id) def _do_test_find_resources(self): with self.assertRaises(BadRequest) as cm: self.resource_registry_service.find_resources(RT.UserInfo, LCS.DRAFT, "name", False) self.assertTrue(cm.exception.message == "find by name does not support lcstate") ret = self.resource_registry_service.find_resources(RT.UserInfo, None, "name", False) self.assertEquals(len(ret[0]), 0) # Instantiate an object obj = IonObject("InstrumentAgentInstance", name="name") # Persist object and read it back obj_id, obj_rev = self.resource_registry_service.create(obj) read_obj = self.resource_registry_service.read(obj_id) ret = self.resource_registry_service.find_resources(RT.InstrumentAgentInstance, None, "name", False) self.assertEquals(len(ret[0]), 1) self.assertEquals(ret[0][0]._id, read_obj._id) ret = self.resource_registry_service.find_resources(RT.InstrumentAgentInstance, LCS.DEPLOYED, None, False) self.assertEquals(len(ret[0]), 1) self.assertEquals(ret[0][0]._id, read_obj._id) def _do_test_find_objects_mult(self): dp = DataProcess() transform = Transform() pd = ProcessDefinition() dp_id, _ = self.resource_registry_service.create(dp) transform_id, _ = self.resource_registry_service.create(transform) pd_id, _ = self.resource_registry_service.create(pd) self.resource_registry_service.create_association(subject=dp_id, object=transform_id, predicate=PRED.hasTransform) self.resource_registry_service.create_association(subject=transform_id, object=pd_id, predicate=PRED.hasProcessDefinition) results, _ = self.resource_registry_service.find_objects_mult(subjects=[dp_id],id_only=True) self.assertTrue(results == [transform_id]) results, _ = self.resource_registry_service.find_objects_mult(subjects=[dp_id, transform_id], id_only=True) results.sort() correct = [transform_id, pd_id] correct.sort() self.assertTrue(results == correct) @attr('EXT') def test_get_resource_extension(self): #Testing multiple instrument owners subject1 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254" actor_identity_obj1 = IonObject(RT.ActorIdentity, {"name": subject1}) actor_id1,_ = self.resource_registry_service.create(actor_identity_obj1) user_info_obj1 = IonObject(RT.UserInfo, {"name": "Foo"}) user_info_id1,_ = self.resource_registry_service.create(user_info_obj1) self.resource_registry_service.create_association(actor_id1, PRED.hasInfo, user_info_id1) subject2 = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256" actor_identity_obj2 = IonObject(RT.ActorIdentity, {"name": subject2}) actor_id2,_ = self.resource_registry_service.create(actor_identity_obj2) user_info_obj2 = IonObject(RT.UserInfo, {"name": "Foo2"}) user_info_id2,_ = self.resource_registry_service.create(user_info_obj2) self.resource_registry_service.create_association(actor_id2, PRED.hasInfo, user_info_id2) test_obj = IonObject(RT.InformationResource, {"name": "TestResource"}) test_obj_id,_ = self.resource_registry_service.create(test_obj) self.resource_registry_service.create_association(test_obj_id, PRED.hasOwner, actor_id1) self.resource_registry_service.create_association(test_obj_id, PRED.hasOwner, actor_id2) extended_resource = self.resource_registry_service.get_resource_extension(test_obj_id, OT.ExtendedInformationResource ) self.assertEqual(test_obj_id,extended_resource._id) self.assertEqual(len(extended_resource.owners),2) extended_resource_list = self.resource_registry_service.get_resource_extension(str([user_info_id1,user_info_id2]), OT.ExtendedInformationResource) self.assertEqual(len(extended_resource_list), 2) optional_args = {'user_id': user_info_id1} extended_resource = self.resource_registry_service.get_resource_extension(test_obj_id, OT.TestExtendedInformationResource, optional_args=optional_args ) self.assertEqual(test_obj_id,extended_resource._id) self.assertEqual(len(extended_resource.owners),2) self.assertEqual(extended_resource.user_id, user_info_id1)