class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dmpsclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.psmsclient = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.resource_impl = ResourceImpl(self.c) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) ) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.resource_impl.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: deployment_obj = self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_activate_deployment(self): #------------------------------------------------------------------------------------- # Create platform site, platform device, platform model #------------------------------------------------------------------------------------- platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device(platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') model_id = self.imsclient.create_platform_model(platform_model__obj) #------------------------------------------------------------------------------------- # Assign platform model to platform device and site #------------------------------------------------------------------------------------- self.imsclient.assign_platform_model_to_platform_device(model_id, platform_device_id) self.omsclient.assign_platform_model_to_platform_site(model_id, site_id) #------------------------------------------------------------------------------------- # Create instrument site #------------------------------------------------------------------------------------- instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='Log Data Product', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) out_log_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id) #---------------------------------------------------------------------------------------------------- # Start the transform (a logical transform) that acts as an instrument site #---------------------------------------------------------------------------------------------------- self.omsclient.create_site_data_product( site_id= instrument_site_id, data_product_id = out_log_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument device #---------------------------------------------------------------------------------------------------- instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) dp_obj = IonObject(RT.DataProduct, name='Instrument Data Product', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) inst_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id) #assign data products appropriately self.damsclient.assign_data_product(input_resource_id=instrument_device_id, data_product_id=inst_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument model #---------------------------------------------------------------------------------------------------- instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj) self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(instrument_model_id, instrument_site_id) #---------------------------------------------------------------------------------------------------- # Create a deployment object #---------------------------------------------------------------------------------------------------- start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instrument_device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dmpsclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.psmsclient = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) ) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.RR2.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self): #------------------------------------------------------------------------------------- # Create platform site, platform device, platform model #------------------------------------------------------------------------------------- platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') platform_site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device(platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model(platform_model__obj) #------------------------------------------------------------------------------------- # Create instrument site #------------------------------------------------------------------------------------- instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) #---------------------------------------------------------------------------------------------------- # Create an instrument device #---------------------------------------------------------------------------------------------------- instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) #---------------------------------------------------------------------------------------------------- # Create an instrument model #---------------------------------------------------------------------------------------------------- instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj) #---------------------------------------------------------------------------------------------------- # Create a deployment object #---------------------------------------------------------------------------------------------------- start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_platform_model_to_platform_device(res.platform_model_id, res.platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_platform_model_to_platform_site(res.platform_model_id, res.platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("adding instrument site and device to deployment") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("adding platform site and device to deployment") self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id) self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) log.debug("deactivatin deployment, expecting success") self.omsclient.deactivate_deployment(res.deployment_id) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without site+device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without instrument site, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "Devices in this deployment outnumber sites") #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) log.debug("activating deployment without device, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment") def test_activate_deployment_asymmetric_children(self): """ P0 | \ P1 P2 | I1 Complex deployment using CSP P1, P2, and P3 share the same platform model. The CSP solver should be able to work this out based on relationships to parents """ log.debug("create models") imodel_id = self.RR2.create(any_old(RT.InstrumentModel)) pmodel_id = self.RR2.create(any_old(RT.PlatformModel)) log.debug("create devices") idevice_id = self.RR2.create(any_old(RT.InstrumentDevice)) pdevice_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(3)] log.debug("create sites") isite_id = self.RR2.create(any_old(RT.InstrumentSite)) psite_id = [self.RR2.create(any_old(RT.PlatformSite)) for _ in range(3)] log.debug("assign models") self.RR2.assign_instrument_model_to_instrument_device_with_has_model(imodel_id, idevice_id) self.RR2.assign_instrument_model_to_instrument_site_with_has_model(imodel_id, isite_id) for x in range(3): self.RR2.assign_platform_model_to_platform_device_with_has_model(pmodel_id, pdevice_id[x]) self.RR2.assign_platform_model_to_platform_site_with_has_model(pmodel_id, psite_id[x]) log.debug("assign hierarchy") self.RR2.assign_instrument_device_to_platform_device_with_has_device(idevice_id, pdevice_id[1]) self.RR2.assign_instrument_site_to_platform_site_with_has_site(isite_id, psite_id[1]) for x in range(1,3): self.RR2.assign_platform_device_to_platform_device_with_has_device(pdevice_id[x], pdevice_id[0]) self.RR2.assign_platform_site_to_platform_site_with_has_site(psite_id[x], psite_id[0]) log.debug("create and activate deployment") dep_id = self.RR2.create(any_old(RT.Deployment, {"context": IonObject(OT.RemotePlatformDeploymentContext)})) self.RR2.assign_deployment_to_platform_device_with_has_deployment(dep_id, pdevice_id[0]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(dep_id, psite_id[0]) self.omsclient.activate_deployment(dep_id) log.debug("verifying deployment") self.assertEqual(idevice_id, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(isite_id), "The instrument device was not assigned to the instrument site") for x in range(3): self.assertEqual(pdevice_id[x], self.RR2.find_platform_device_id_of_platform_site_using_has_device(psite_id[x]), "Platform device %d was not assigned to platform site %d" % (x, x)) def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message="did not specify fail_message"): with self.assertRaises(err_type) as cm: self.omsclient.activate_deployment(deployment_id) self.assertIn(fail_message, cm.exception.message) def test_3x3_matchups_remoteplatform(self): self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext)) def test_3x3_matchups_cabledinstrument(self): self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext)) def test_3x3_matchups_cablednode(self): self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext)) def base_3x3_matchups(self, deployment_context): """ This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1) """ deployment_context_type = type(deployment_context).__name__ instrument_model_id = [self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)] platform_model_id = [self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)] instrument_device_id = [self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)] platform_device_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)] instrument_site_id = [self.RR2.create(any_old(RT.InstrumentSite, {"planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="instport_%d" % (i+1))})) for i in range(9)] platform_site_id = [self.RR2.create(any_old(RT.PlatformSite, {"planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="platport_%d" % (i+1))})) for i in range(4)] def instrument_model_at(platform_idx, instrument_idx): m = platform_idx * 2 if instrument_idx > 0: m += 1 return m def platform_model_at(platform_idx): if platform_idx > 0: return 1 return 0 def instrument_at(platform_idx, instrument_idx): return platform_idx * 3 + instrument_idx # set up the structure for p in range(3): m = platform_model_at(p) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[m], platform_site_id[p]) self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[m], platform_device_id[p]) self.RR2.assign_platform_device_to_platform_device_with_has_device(platform_device_id[p], platform_device_id[3]) self.RR2.assign_platform_site_to_platform_site_with_has_site(platform_site_id[p], platform_site_id[3]) for i in range(3): m = instrument_model_at(p, i) idx = instrument_at(p, i) self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id[m], instrument_site_id[idx]) self.RR2.assign_instrument_model_to_instrument_device_with_has_model(instrument_model_id[m], instrument_device_id[idx]) self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id[idx], platform_device_id[p]) self.RR2.assign_instrument_site_to_platform_site_with_has_site(instrument_site_id[idx], platform_site_id[p]) # top level models self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[2], platform_device_id[3]) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[2], platform_site_id[3]) # verify structure for p in range(3): parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(platform_device_id[p]) self.assertEqual(platform_device_id[3], parent_id) parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(platform_site_id[p]) self.assertEqual(platform_site_id[3], parent_id) for i in range(len(platform_site_id)): self.assertEqual(self.RR2.find_platform_model_of_platform_device_using_has_model(platform_device_id[i]), self.RR2.find_platform_model_of_platform_site_using_has_model(platform_site_id[i])) for i in range(len(instrument_site_id)): self.assertEqual(self.RR2.find_instrument_model_of_instrument_device_using_has_model(instrument_device_id[i]), self.RR2.find_instrument_model_of_instrument_site_using_has_model(instrument_site_id[i])) port_assignments = {} for p in range(3): port_assignments[platform_device_id[p]] = "platport_%d" % (p+1) for i in range(3): idx = instrument_at(p, i) port_assignments[instrument_device_id[idx]] = "instport_%d" % (idx+1) deployment_id = self.RR2.create(any_old(RT.Deployment, {"context": deployment_context, "port_assignments": port_assignments})) log.debug("assigning device/site to %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[3]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[3]) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_instrument_device_with_has_deployment(deployment_id, instrument_device_id[1]) self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id[1]) elif OT.CabledNodeDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[1]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[1]) log.debug("activation of %s deployment", deployment_context_type) self.omsclient.activate_deployment(deployment_id) log.debug("validation of %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(d, self.RR2.find_platform_device_id_of_platform_site_using_has_device(platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual(d, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[i])) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.assertEqual(instrument_device_id[1], self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[1])) elif OT.CabledNodeDeploymentContext == deployment_context_type: expected_platforms = [1] expected_instruments = [3, 4, 5] # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(i in expected_platforms, d in self.RR2.find_platform_device_ids_of_platform_site_using_has_device(platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual(i in expected_instruments, d in self.RR2.find_instrument_device_ids_of_instrument_site_using_has_device(instrument_site_id[i]))
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() # self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.container.start_rel_from_url("res/deploy/r2deploy.yml") print "started services" # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=""): # logger process producer_definition = ProcessDefinition(name=name + "_logger") producer_definition.executable = { "module": "ion.processes.data.stream_granule_logger", "class": "StreamGranuleLogger", } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = {"process": {"stream_id": stream_id}} pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration ) return pid def cleanupprocs(self): stm = os.popen("ps -e | grep ion.agents.port.logger_process") procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) stm = os.popen("ps -e | grep ion.agents.instrument.zmq_driver_process") procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) # stm = os.popen('rm /tmp/*.pid.txt') @unittest.skip("Deprecated by IngestionManagement refactor, timeout on start inst agent?") def test_deploy_activate_full(self): # ensure no processes or pids are left around by agents or Sims # self.cleanupprocs() self.loggerpids = [] # ------------------------------- # Create InstrumentModel # ------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) # ------------------------------- # Create InstrumentAgent # ------------------------------- instAgent_obj = IonObject( RT.InstrumentAgent, name="agent007", description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.debug("new InstrumentAgent id = %s", instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # ------------------------------- # Create Instrument Site # ------------------------------- instrumentSite_obj = IonObject(RT.InstrumentSite, name="instrumentSite1", description="SBE37IMInstrumentSite") try: instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id="") except BadRequest as ex: self.fail("failed to create new InstrumentSite: %s" % ex) print "test_deployAsPrimaryDevice: new instrumentSite id = ", instrumentSite_id self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id) # ------------------------------- # Logical Transform: Output Data Products # ------------------------------- # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( "ctd_parsed_param_dict", id_only=True ) parsed_stream_def_id = self.pubsubclient.create_stream_definition( name="parsed", parameter_dictionary_id=parsed_pdict_id ) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_raw_param_dict", id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition(name="raw", parameter_dictionary_id=raw_pdict_id) # ------------------------------- # Create Old InstrumentDevice # ------------------------------- instDevice_obj = IonObject( RT.InstrumentDevice, name="SBE37IMDeviceYear1", description="SBE37IMDevice for the FIRST year of deployment", serial_number="12345", ) try: oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print "test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ", oldInstDevice_id self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE) # ------------------------------- # Create Raw and Parsed Data Products for the device # ------------------------------- dp_obj = IonObject( RT.DataProduct, name="SiteDataProduct", description="SiteDataProduct", temporal_domain=tdom, spatial_domain=sdom, ) instrument_site_output_dp_id = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id ) self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=instrument_site_output_dp_id ) # self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasStream, None, True) log.debug("Data product streams1 = %s", stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True) log.debug("Data set for data_product_id1 = %s", dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger("ctd_parsed", stream_ids[0]) self.loggerpids.append(pid) self.omsclient.create_site_data_product(instrumentSite_id, instrument_site_output_dp_id) # ------------------------------- # Create Old Deployment # ------------------------------- deployment_obj = IonObject(RT.Deployment, name="first deployment") oldDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id) # ------------------------------- # Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information # cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, # ------------------------------- port_agent_config = { "device_addr": CFG.device.sbe37.host, "device_port": CFG.device.sbe37.port, "process_type": PortAgentProcessType.UNIX, "binary_path": "port_agent", "port_agent_addr": "localhost", "command_port": CFG.device.sbe37.port_agent_cmd_port, "data_port": CFG.device.sbe37.port_agent_data_port, "log_level": 5, "type": PortAgentType.ETHERNET, } raw_config = StreamConfiguration( stream_name="raw", parameter_dictionary_name="ctd_raw_param_dict", records_per_granule=2, granule_publish_rate=5, ) parsed_config = StreamConfiguration( stream_name="parsed", parameter_dictionary_name="ctd_parsed_param_dict", records_per_granule=2, granule_publish_rate=5, ) instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstanceYear1", description="SBE37IMAgentInstanceYear1", comms_device_address="sbe37-simulator.oceanobservatories.org", comms_device_port=4001, port_agent_config=port_agent_config, stream_configurations=[raw_config, parsed_config], ) oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, oldInstDevice_id ) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() # ------------------------------- # Create CTD Parsed as the Year 1 data product and attach to instrument # ------------------------------- print "Creating new CDM data product with a stream definition" dp_obj = IonObject( RT.DataProduct, name="ctd_parsed_year1", description="ctd stream test year 1", temporal_domain=tdom, spatial_domain=sdom, ) ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id ) print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year1 self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1 ) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True) print "test_deployAsPrimaryDevice: Data product streams1 = ", stream_ids # ------------------------------- # Create New InstrumentDevice # ------------------------------- instDevice_obj_2 = IonObject( RT.InstrumentDevice, name="SBE37IMDeviceYear2", description="SBE37IMDevice for the SECOND year of deployment", serial_number="67890", ) try: newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print "test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ", newInstDevice_id # set the LCSTATE self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE) instDevice_obj_2 = self.rrclient.read(newInstDevice_id) log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate)) # ------------------------------- # Create Old Deployment # ------------------------------- deployment_obj = IonObject(RT.Deployment, name="second deployment") newDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id) # ------------------------------- # Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information # ------------------------------- port_agent_config = { "device_addr": "sbe37-simulator.oceanobservatories.org", "device_port": 4004, "process_type": PortAgentProcessType.UNIX, "binary_path": "port_agent", "port_agent_addr": "localhost", "command_port": 4005, "data_port": 4006, "log_level": 5, "type": PortAgentType.ETHERNET, } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstanceYear2", description="SBE37IMAgentInstanceYear2", comms_device_address="sbe37-simulator.oceanobservatories.org", comms_device_port=4004, port_agent_config=port_agent_config, ) newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, newInstDevice_id ) # ------------------------------- # Create CTD Parsed as the Year 2 data product # ------------------------------- dp_obj = IonObject( RT.DataProduct, name="ctd_parsed_year2", description="ctd stream test year 2", temporal_domain=tdom, spatial_domain=sdom, ) ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id ) print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year2 self.damsclient.assign_data_product( input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2 ) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True) print "test_deployAsPrimaryDevice: Data product streams2 = ", stream_ids # ------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition # ------------------------------- log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all") dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L0_all", description="transform ctd package into three separate L0 streams", module="ion.processes.data.transforms.ctd.ctd_L0_all", class_name="ctd_L0_all", ) try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" % ex) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products # ------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name="L0_Conductivity", parameter_dictionary_id=parsed_pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding="conductivity" ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name="L0_Pressure", parameter_dictionary_id=parsed_pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding="pressure" ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name="L0_Temperature", parameter_dictionary_id=parsed_pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding="temperature" ) self.output_products = {} log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name="L0_Conductivity", description="transform output conductivity", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_conductivity_output_dp_obj, stream_definition_id=parsed_stream_def_id ) self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name="L0_Pressure", description="transform output pressure", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_pressure_output_dp_obj, stream_definition_id=parsed_stream_def_id ) self.output_products["pressure"] = ctd_l0_pressure_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name="L0_Temperature", description="transform output temperature", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_temperature_output_dp_obj, stream_definition_id=parsed_stream_def_id ) self.output_products["temperature"] = ctd_l0_temperature_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product) # ------------------------------- log.debug("test_deployAsPrimaryDevice: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products ) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug("test_deployAsPrimaryDevice: create L0 all data_process return") # -------------------------------- # Activate the deployment # -------------------------------- self.omsclient.activate_deployment(oldDeployment_id) # ------------------------------- # Launch InstrumentAgentInstance Sim1, connect to the resource agent client # ------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id) self.addCleanup( self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=oldInstAgentInstance_id ) # wait for start instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) gate = ProcessStateGate( self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING ) self.assertTrue( gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id, ) inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent1_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim1 = ResourceAgentClient( "iaclient Sim1", name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess() ) print "activate_instrument: got _ia_client_sim1 %s", self._ia_client_sim1 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1)) # ------------------------------- # Launch InstrumentAgentInstance Sim2, connect to the resource agent client # ------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id) self.addCleanup( self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=newInstAgentInstance_id ) # wait for start instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) gate = ProcessStateGate( self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING ) self.assertTrue( gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id, ) inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent2_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim2 = ResourceAgentClient( "iaclient Sim2", name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess() ) print "activate_instrument: got _ia_client_sim2 %s", self._ia_client_sim2 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2)) # ------------------------------- # Streaming Sim1 (old instrument) # ------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active %s", str(reply)) self.assertTrue(reply) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim1.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE: %s", str(retval)) # ------------------------------- # Streaming Sim 2 (new instrument) # ------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active_sim2 %s", str(reply)) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim2.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s", str(retval)) gevent.sleep(10) # ------------------------------- # Shutdown Sim1 (old instrument) # ------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(5) # ------------------------------- # Shutdown Sim2 (old instrument) # ------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset_sim2 ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset_sim2 %s", str(reply)) time.sleep(5)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dmpsclient = DataProductManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.psmsclient = PubsubManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.dsmsclient = DataProcessManagementServiceClient( node=self.container.node) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources( RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj)) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_prepare_deployment_support(self): deploy_sup = self.omsclient.prepare_deployment_support() self.assertTrue(deploy_sup) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].type_, "AssocDeploymentInstDevice") self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice'].type_, "AssocDeploymentPlatDevice") self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].type_, "AssocDeploymentInstSite") self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite'].type_, "AssocDeploymentPlatSite") self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite']. associated_resources, []) #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformDevice']. resources), 1) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformSite'].resources ), 1) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite']. associated_resources, []) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformDevice']. resources), 1) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformDevice']. associated_resources), 1) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformSite'].resources ), 1) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformSite']. associated_resources), 1) #delete the deployment self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self, make_assigns=False): # Create platform site, platform device, platform model bounds = GeospatialBounds(geospatial_latitude_limit_north=float(5), geospatial_latitude_limit_south=float(5), geospatial_longitude_limit_west=float(15), geospatial_longitude_limit_east=float(15), geospatial_vertical_min=float(0), geospatial_vertical_max=float(1000)) platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site', constraint_list=[bounds]) platform_site_id = self.omsclient.create_platform_site( platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device( platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model( platform_model__obj) # Create instrument site #------------------------------------------------------------------------------------- bounds = GeospatialBounds(geospatial_latitude_limit_north=float(45), geospatial_latitude_limit_south=float(40), geospatial_longitude_limit_west=float(-75), geospatial_longitude_limit_east=float(-70), geospatial_vertical_min=float(0), geospatial_vertical_max=float(500)) instrument_site_obj = IonObject( RT.InstrumentSite, name='InstrumentSite1', description='test instrument site', reference_designator='GA01SUMO-FI003-01-CTDMO0999', constraint_list=[bounds]) instrument_site_id = self.omsclient.create_instrument_site( instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) # Create an instrument device instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) pp_obj = IonObject(OT.PlatformPort, reference_designator='GA01SUMO-FI003-01-CTDMO0999', port_type=PortTypeEnum.PAYLOAD, ip_address='1') port_assignments = {instrument_device_id: pp_obj} #---------------------------------------------------------------------------------------------------- # Create an instrument model instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model( instrument_model_obj) # Create a deployment object #---------------------------------------------------------------------------------------------------- start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2020, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', context=IonObject( OT.CabledNodeDeploymentContext), port_assignments=port_assignments, constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) if make_assigns: self.imsclient.assign_platform_model_to_platform_device( platform_model_id, platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device( instrument_model_id, instrument_device_id) self.omsclient.assign_platform_model_to_platform_site( platform_model_id, platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site( instrument_model_id, instrument_site_id) self.omsclient.assign_site_to_deployment(platform_site_id, deployment_id) self.omsclient.assign_device_to_deployment(platform_device_id, deployment_id) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret def _create_subsequent_deployment(self, prior_dep_info): platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice2', description='test platform device') platform_device_id = self.imsclient.create_platform_device( platform_device_obj) instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice2', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.imsclient.assign_platform_model_to_platform_device( prior_dep_info.platform_model_id, platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device( prior_dep_info.instrument_model_id, instrument_device_id) start = str(int(time.mktime(datetime.datetime(2013, 6, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2020, 6, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment2', description='some new deployment', context=IonObject( OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment( prior_dep_info.platform_site_id, deployment_id) self.omsclient.assign_device_to_deployment( prior_dep_info.platform_device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) ret = DotDict(instrument_device_id=instrument_device_id, platform_device_id=platform_device_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment(make_assigns=True) before_activate_instrument_device_obj = self.rrclient.read( res.instrument_device_id) self.assertNotEquals(before_activate_instrument_device_obj.lcstate, LCS.DEPLOYED) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) def assertGeospatialBoundsEquals(a, b): self.assertEquals(a['geospatial_latitude_limit_north'], b['geospatial_latitude_limit_north']) self.assertEquals(a['geospatial_latitude_limit_south'], b['geospatial_latitude_limit_south']) self.assertEquals(a['geospatial_longitude_limit_west'], b['geospatial_longitude_limit_west']) self.assertEquals(a['geospatial_longitude_limit_east'], b['geospatial_longitude_limit_east']) def assertGeospatialBoundsNotEquals(a, b): self.assertNotEquals(a['geospatial_latitude_limit_north'], b['geospatial_latitude_limit_north']) self.assertNotEquals(a['geospatial_latitude_limit_south'], b['geospatial_latitude_limit_south']) self.assertNotEquals(a['geospatial_longitude_limit_west'], b['geospatial_longitude_limit_west']) self.assertNotEquals(a['geospatial_longitude_limit_east'], b['geospatial_longitude_limit_east']) after_activate_instrument_device_obj = self.rrclient.read( res.instrument_device_id) assertGeospatialBoundsNotEquals( before_activate_instrument_device_obj.geospatial_bounds, after_activate_instrument_device_obj.geospatial_bounds) deployment_obj = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj.lcstate, LCS.DEPLOYED) log.debug("deactivatin deployment, expecting success") self.omsclient.deactivate_deployment(res.deployment_id) after_deactivate_instrument_device_obj = self.rrclient.read( res.instrument_device_id) assertGeospatialBoundsNotEquals( after_activate_instrument_device_obj.geospatial_bounds, after_deactivate_instrument_device_obj.geospatial_bounds) deployment_obj = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj.lcstate, LCS.INTEGRATED) def test_activate_deployment_redeploy(self): dep_util = DeploymentUtil(self.container) res = self.base_activate_deployment(make_assigns=True) log.debug("activating first deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) deployment_obj1 = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj1.lcstate, LCS.DEPLOYED) next_dep_info = self._create_subsequent_deployment(res) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertNotEquals(deployment_obj2.lcstate, LCS.DEPLOYED) log.debug("activating subsequent deployment, expecting success") self.omsclient.activate_deployment(next_dep_info.deployment_id) deployment_obj1 = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj1.lcstate, LCS.INTEGRATED) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertEquals(deployment_obj2.lcstate, LCS.DEPLOYED) dep1_tc = dep_util.get_temporal_constraint(deployment_obj1) dep2_tc = dep_util.get_temporal_constraint(deployment_obj2) self.assertLessEqual(float(dep1_tc.end_datetime), float(dep2_tc.end_datetime)) log.debug("deactivating second deployment, expecting success") self.omsclient.deactivate_deployment(next_dep_info.deployment_id) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertEquals(deployment_obj2.lcstate, LCS.INTEGRATED) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.assign_site_to_deployment(res.platform_site_id, res.deployment_id) self.omsclient.assign_device_to_deployment(res.platform_device_id, res.deployment_id) log.debug( "activating deployment without site+device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug( "activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.omsclient.assign_device_to_deployment(res.instrument_device_id, res.deployment_id) log.debug( "activating deployment without instrument site, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest) #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.assign_site_to_deployment(res.instrument_site_id, res.deployment_id) log.debug("activating deployment without device, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment") def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message=""): with self.assertRaises(err_type) as cm: self.omsclient.activate_deployment(deployment_id) log.debug("assert_deploy_fail cm: %s", str(cm)) if fail_message: self.assertIn(fail_message, cm.exception.message) def test_3x3_matchups_remoteplatform(self): self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext)) def test_3x3_matchups_cabledinstrument(self): self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext)) def test_3x3_matchups_cablednode(self): self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext)) def base_3x3_matchups(self, deployment_context): """ This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1) """ deployment_context_type = type(deployment_context).__name__ instrument_model_id = [ self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6) ] platform_model_id = [ self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3) ] instrument_device_id = [ self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9) ] platform_device_id = [ self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4) ] instrument_site_id = [ self.RR2.create( any_old( RT.InstrumentSite, { "reference_designator": "GA01SUMO-FI003-0%s-CTDMO0999" % (i + 1), "planned_uplink_port": IonObject( OT.PlatformPort, reference_designator="GA01SUMO-FI003-0%s-CTDMO0999" % (i + 1)) })) for i in range(9) ] platform_site_id = [ self.RR2.create( any_old( RT.PlatformSite, { "reference_designator": "GA01SUMO-FI003-0%s-CTDMO0888" % (i + 1), "planned_uplink_port": IonObject( OT.PlatformPort, reference_designator="GA01SUMO-FI003-0%s-CTDMO0888" % (i + 1)) })) for i in range(4) ] def instrument_model_at(platform_idx, instrument_idx): m = platform_idx * 2 if instrument_idx > 0: m += 1 return m def platform_model_at(platform_idx): if platform_idx > 0: return 1 return 0 def instrument_at(platform_idx, instrument_idx): return platform_idx * 3 + instrument_idx # set up the structure for p in range(3): m = platform_model_at(p) self.RR2.assign_platform_model_to_platform_site_with_has_model( platform_model_id[m], platform_site_id[p]) self.RR2.assign_platform_model_to_platform_device_with_has_model( platform_model_id[m], platform_device_id[p]) self.RR2.assign_platform_device_to_platform_device_with_has_device( platform_device_id[p], platform_device_id[3]) self.RR2.assign_platform_site_to_platform_site_with_has_site( platform_site_id[p], platform_site_id[3]) for i in range(3): m = instrument_model_at(p, i) idx = instrument_at(p, i) self.RR2.assign_instrument_model_to_instrument_site_with_has_model( instrument_model_id[m], instrument_site_id[idx]) self.RR2.assign_instrument_model_to_instrument_device_with_has_model( instrument_model_id[m], instrument_device_id[idx]) self.RR2.assign_instrument_device_to_platform_device_with_has_device( instrument_device_id[idx], platform_device_id[p]) self.RR2.assign_instrument_site_to_platform_site_with_has_site( instrument_site_id[idx], platform_site_id[p]) # top level models self.RR2.assign_platform_model_to_platform_device_with_has_model( platform_model_id[2], platform_device_id[3]) self.RR2.assign_platform_model_to_platform_site_with_has_model( platform_model_id[2], platform_site_id[3]) # verify structure for p in range(3): parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device( platform_device_id[p]) self.assertEqual(platform_device_id[3], parent_id) parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site( platform_site_id[p]) self.assertEqual(platform_site_id[3], parent_id) for i in range(len(platform_site_id)): self.assertEqual( self.RR2. find_platform_model_of_platform_device_using_has_model( platform_device_id[i]), self.RR2.find_platform_model_of_platform_site_using_has_model( platform_site_id[i])) for i in range(len(instrument_site_id)): self.assertEqual( self.RR2. find_instrument_model_of_instrument_device_using_has_model( instrument_device_id[i]), self.RR2. find_instrument_model_of_instrument_site_using_has_model( instrument_site_id[i])) # OOIReferenceDesignator format: GA01SUMO-FI003-03-CTDMO0999 (site-platform_id-port-device_id) port_assignments = {} for p in range(3): ref_desig = "GA01SUMO-FI003-0%s-CTDMO0888" % (p + 1) pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type=PortTypeEnum.PAYLOAD, ip_address=str(p)) port_assignments[platform_device_id[p]] = pp_obj for i in range(3): ref_desig = "GA01SUMO-FI003-0%s-CTDMO0999" % ((p * 3) + i + 1) pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type=PortTypeEnum.PAYLOAD, ip_address=str(p)) idx = instrument_at(p, i) port_assignments[instrument_device_id[idx]] = pp_obj deployment_id = self.RR2.create( any_old( RT.Deployment, { "context": deployment_context, "port_assignments": port_assignments })) log.debug("assigning device/site to %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment( deployment_id, platform_device_id[3]) self.RR2.assign_deployment_to_platform_site_with_has_deployment( deployment_id, platform_site_id[3]) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_instrument_device_with_has_deployment( deployment_id, instrument_device_id[1]) self.RR2.assign_deployment_to_instrument_site_with_has_deployment( deployment_id, instrument_site_id[1]) elif OT.CabledNodeDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment( deployment_id, platform_device_id[1]) self.RR2.assign_deployment_to_platform_site_with_has_deployment( deployment_id, platform_site_id[1]) log.debug("activation of %s deployment", deployment_context_type) self.omsclient.activate_deployment(deployment_id) log.debug("validation of %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual( d, self.RR2. find_platform_device_id_of_platform_site_using_has_device( platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual( d, self.RR2. find_instrument_device_id_of_instrument_site_using_has_device( instrument_site_id[i])) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.assertEqual( instrument_device_id[1], self.RR2. find_instrument_device_id_of_instrument_site_using_has_device( instrument_site_id[1])) elif OT.CabledNodeDeploymentContext == deployment_context_type: expected_platforms = [1] # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual( i in expected_platforms, d in self.RR2. find_platform_device_ids_of_platform_site_using_has_device( platform_site_id[i]))
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dmpsclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.psmsclient = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) ) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.RR2.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_prepare_deployment_support(self): deploy_sup = self.omsclient.prepare_deployment_support() self.assertTrue(deploy_sup) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].type_, "AssocDeploymentInstDevice") self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].type_, "AssocDeploymentPlatDevice") self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].type_, "AssocDeploymentInstSite") self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].type_, "AssocDeploymentPlatSite") self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, []) #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, []) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources), 1) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources), 1) #delete the deployment self.RR2.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self): #------------------------------------------------------------------------------------- # Create platform site, platform device, platform model #------------------------------------------------------------------------------------- platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') platform_site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device(platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model(platform_model__obj) #------------------------------------------------------------------------------------- # Create instrument site #------------------------------------------------------------------------------------- instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) #---------------------------------------------------------------------------------------------------- # Create an instrument device #---------------------------------------------------------------------------------------------------- instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) #---------------------------------------------------------------------------------------------------- # Create an instrument model #---------------------------------------------------------------------------------------------------- instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj) #---------------------------------------------------------------------------------------------------- # Create a deployment object #---------------------------------------------------------------------------------------------------- start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_platform_model_to_platform_device(res.platform_model_id, res.platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_platform_model_to_platform_site(res.platform_model_id, res.platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("adding instrument site and device to deployment") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("adding platform site and device to deployment") self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id) self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) log.debug("deactivatin deployment, expecting success") self.omsclient.deactivate_deployment(res.deployment_id) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without site+device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without instrument site, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "Devices in this deployment outnumber sites") #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) log.debug("activating deployment without device, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment") def test_activate_deployment_asymmetric_children(self): """ P0 | \ P1 P2 | I1 Complex deployment using CSP P1, P2, and P3 share the same platform model. The CSP solver should be able to work this out based on relationships to parents """ log.debug("create models") imodel_id = self.RR2.create(any_old(RT.InstrumentModel)) pmodel_id = self.RR2.create(any_old(RT.PlatformModel)) log.debug("create devices") idevice_id = self.RR2.create(any_old(RT.InstrumentDevice)) pdevice_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(3)] log.debug("create sites") isite_id = self.RR2.create(any_old(RT.InstrumentSite)) psite_id = [self.RR2.create(any_old(RT.PlatformSite)) for _ in range(3)] log.debug("assign models") self.RR2.assign_instrument_model_to_instrument_device_with_has_model(imodel_id, idevice_id) self.RR2.assign_instrument_model_to_instrument_site_with_has_model(imodel_id, isite_id) for x in range(3): self.RR2.assign_platform_model_to_platform_device_with_has_model(pmodel_id, pdevice_id[x]) self.RR2.assign_platform_model_to_platform_site_with_has_model(pmodel_id, psite_id[x]) log.debug("assign hierarchy") self.RR2.assign_instrument_device_to_platform_device_with_has_device(idevice_id, pdevice_id[1]) self.RR2.assign_instrument_site_to_platform_site_with_has_site(isite_id, psite_id[1]) for x in range(1,3): self.RR2.assign_platform_device_to_platform_device_with_has_device(pdevice_id[x], pdevice_id[0]) self.RR2.assign_platform_site_to_platform_site_with_has_site(psite_id[x], psite_id[0]) log.debug("create and activate deployment") dep_id = self.RR2.create(any_old(RT.Deployment, {"context": IonObject(OT.RemotePlatformDeploymentContext)})) self.RR2.assign_deployment_to_platform_device_with_has_deployment(dep_id, pdevice_id[0]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(dep_id, psite_id[0]) self.omsclient.activate_deployment(dep_id) log.debug("verifying deployment") self.assertEqual(idevice_id, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(isite_id), "The instrument device was not assigned to the instrument site") for x in range(3): self.assertEqual(pdevice_id[x], self.RR2.find_platform_device_id_of_platform_site_using_has_device(psite_id[x]), "Platform device %d was not assigned to platform site %d" % (x, x)) def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message="did not specify fail_message"): with self.assertRaises(err_type) as cm: self.omsclient.activate_deployment(deployment_id) self.assertIn(fail_message, cm.exception.message) def test_3x3_matchups_remoteplatform(self): self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext)) def test_3x3_matchups_cabledinstrument(self): self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext)) def test_3x3_matchups_cablednode(self): self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext)) def base_3x3_matchups(self, deployment_context): """ This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1) """ deployment_context_type = type(deployment_context).__name__ instrument_model_id = [self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)] platform_model_id = [self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)] instrument_device_id = [self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)] platform_device_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)] instrument_site_id = [self.RR2.create(any_old(RT.InstrumentSite, {"planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="instport_%d" % (i+1))})) for i in range(9)] platform_site_id = [self.RR2.create(any_old(RT.PlatformSite, {"planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="platport_%d" % (i+1))})) for i in range(4)] def instrument_model_at(platform_idx, instrument_idx): m = platform_idx * 2 if instrument_idx > 0: m += 1 return m def platform_model_at(platform_idx): if platform_idx > 0: return 1 return 0 def instrument_at(platform_idx, instrument_idx): return platform_idx * 3 + instrument_idx # set up the structure for p in range(3): m = platform_model_at(p) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[m], platform_site_id[p]) self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[m], platform_device_id[p]) self.RR2.assign_platform_device_to_platform_device_with_has_device(platform_device_id[p], platform_device_id[3]) self.RR2.assign_platform_site_to_platform_site_with_has_site(platform_site_id[p], platform_site_id[3]) for i in range(3): m = instrument_model_at(p, i) idx = instrument_at(p, i) self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id[m], instrument_site_id[idx]) self.RR2.assign_instrument_model_to_instrument_device_with_has_model(instrument_model_id[m], instrument_device_id[idx]) self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id[idx], platform_device_id[p]) self.RR2.assign_instrument_site_to_platform_site_with_has_site(instrument_site_id[idx], platform_site_id[p]) # top level models self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[2], platform_device_id[3]) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[2], platform_site_id[3]) # verify structure for p in range(3): parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(platform_device_id[p]) self.assertEqual(platform_device_id[3], parent_id) parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(platform_site_id[p]) self.assertEqual(platform_site_id[3], parent_id) for i in range(len(platform_site_id)): self.assertEqual(self.RR2.find_platform_model_of_platform_device_using_has_model(platform_device_id[i]), self.RR2.find_platform_model_of_platform_site_using_has_model(platform_site_id[i])) for i in range(len(instrument_site_id)): self.assertEqual(self.RR2.find_instrument_model_of_instrument_device_using_has_model(instrument_device_id[i]), self.RR2.find_instrument_model_of_instrument_site_using_has_model(instrument_site_id[i])) port_assignments = {} for p in range(3): port_assignments[platform_device_id[p]] = "platport_%d" % (p+1) for i in range(3): idx = instrument_at(p, i) port_assignments[instrument_device_id[idx]] = "instport_%d" % (idx+1) deployment_id = self.RR2.create(any_old(RT.Deployment, {"context": deployment_context, "port_assignments": port_assignments})) log.debug("assigning device/site to %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[3]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[3]) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_instrument_device_with_has_deployment(deployment_id, instrument_device_id[1]) self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id[1]) elif OT.CabledNodeDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[1]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[1]) log.debug("activation of %s deployment", deployment_context_type) self.omsclient.activate_deployment(deployment_id) log.debug("validation of %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(d, self.RR2.find_platform_device_id_of_platform_site_using_has_device(platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual(d, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[i])) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.assertEqual(instrument_device_id[1], self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[1])) elif OT.CabledNodeDeploymentContext == deployment_context_type: expected_platforms = [1] expected_instruments = [3, 4, 5] # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(i in expected_platforms, d in self.RR2.find_platform_device_ids_of_platform_site_using_has_device(platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual(i in expected_instruments, d in self.RR2.find_instrument_device_ids_of_instrument_site_using_has_device(instrument_site_id[i]))
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() #@unittest.skip('not ready') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", stream_configuration= {'parsed': 'ctd_parsed_param_dict' } ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4003, 'data_port': 4000, 'log_level': 5, } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) #self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.provenance_product_list.status) log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) #log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(1, len(extended_process_def.data_products) ) #one list because of one data process self.assertEqual(3, len(extended_process_def.data_products[0]) ) #inside that inner list are the three output data products #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() # create missing data process definition dpd_obj = IonObject(RT.DataProcessDefinition, name=LOGICAL_TRANSFORM_DEFINITION_NAME, description="normally in preload", module='ion.processes.data.transforms.logical_transform', class_name='logical_transform') self.dataprocessclient.create_data_process_definition(dpd_obj) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip('not ready') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [parsed_config] ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) self.assertEqual(1, len(extended_product.data_processes) ) self.assertEqual(3, len(extended_product.process_input_data_products) ) # log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) # log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id) #------------------------------- # Cleanup #------------------------------- self.dpmsclient.delete_data_product(ctd_parsed_data_product) self.dpmsclient.delete_data_product(log_data_product_id) self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dmpsclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.psmsclient = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) # create missing data process definition self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node) dpd_obj = IonObject(RT.DataProcessDefinition, name=LOGICAL_TRANSFORM_DEFINITION_NAME, description="normally in preload", module='ion.processes.data.transforms.logical_transform', class_name='logical_transform') self.dsmsclient.create_data_process_definition(dpd_obj) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) ) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.RR2.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self): #------------------------------------------------------------------------------------- # Create platform site, platform device, platform model #------------------------------------------------------------------------------------- platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') platform_site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device(platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model(platform_model__obj) #------------------------------------------------------------------------------------- # Create instrument site #------------------------------------------------------------------------------------- instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='Log Data Product', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) out_log_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id) #---------------------------------------------------------------------------------------------------- # Start the transform (a logical transform) that acts as an instrument site #---------------------------------------------------------------------------------------------------- self.omsclient.create_site_data_product( site_id= instrument_site_id, data_product_id = out_log_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument device #---------------------------------------------------------------------------------------------------- instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) dp_obj = IonObject(RT.DataProduct, name='Instrument Data Product', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) inst_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id) #assign data products appropriately self.damsclient.assign_data_product(input_resource_id=instrument_device_id, data_product_id=inst_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument model #---------------------------------------------------------------------------------------------------- instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj) #---------------------------------------------------------------------------------------------------- # Create a deployment object #---------------------------------------------------------------------------------------------------- start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_platform_model_to_platform_device(res.platform_model_id, res.platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_platform_model_to_platform_site(res.platform_model_id, res.platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("adding instrument site and device to deployment") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("adding platform site and device to deployment") self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id) self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without site+device models, expecting fail") self.assert_deploy_fail(res.deployment_id, "Expected at least 1 model for InstrumentSite") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, "Expected 1 model for InstrumentDevice") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, "No sites were found in the deployment") #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) log.debug("activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, "The set of devices could not be mapped to the set of sites") def assert_deploy_fail(self, deployment_id, fail_message="did not specify fail_message"): with self.assertRaises(BadRequest) as cm: self.omsclient.activate_deployment(deployment_id) self.assertIn(fail_message, cm.exception.message)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dmpsclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.psmsclient = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) ) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_prepare_deployment_support(self): deploy_sup = self.omsclient.prepare_deployment_support() self.assertTrue(deploy_sup) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].type_, "AssocDeploymentInstDevice") self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].type_, "AssocDeploymentPlatDevice") self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].type_, "AssocDeploymentInstSite") self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].type_, "AssocDeploymentPlatSite") self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, []) #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, []) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources), 1) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources), 1) #delete the deployment self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self, make_assigns=False): # Create platform site, platform device, platform model bounds = GeospatialBounds(geospatial_latitude_limit_north=float(5), geospatial_latitude_limit_south=float(5), geospatial_longitude_limit_west=float(15), geospatial_longitude_limit_east=float(15), geospatial_vertical_min=float(0), geospatial_vertical_max=float(1000)) platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site', constraint_list=[bounds]) platform_site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device(platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model(platform_model__obj) # Create instrument site #------------------------------------------------------------------------------------- bounds = GeospatialBounds(geospatial_latitude_limit_north=float(45), geospatial_latitude_limit_south=float(40), geospatial_longitude_limit_west=float(-75), geospatial_longitude_limit_east=float(-70), geospatial_vertical_min=float(0), geospatial_vertical_max=float(500)) instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site', reference_designator='GA01SUMO-FI003-01-CTDMO0999', constraint_list=[bounds]) instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) # Create an instrument device instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) pp_obj = IonObject(OT.PlatformPort, reference_designator='GA01SUMO-FI003-01-CTDMO0999', port_type= PortTypeEnum.PAYLOAD, ip_address='1' ) port_assignments = {instrument_device_id : pp_obj} #---------------------------------------------------------------------------------------------------- # Create an instrument model instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj) # Create a deployment object #---------------------------------------------------------------------------------------------------- start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2020, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), port_assignments=port_assignments, constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) if make_assigns: self.imsclient.assign_platform_model_to_platform_device(platform_model_id, platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) self.omsclient.assign_platform_model_to_platform_site(platform_model_id, platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site(instrument_model_id, instrument_site_id) self.omsclient.assign_site_to_deployment(platform_site_id, deployment_id) self.omsclient.assign_device_to_deployment(platform_device_id, deployment_id) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret def _create_subsequent_deployment(self, prior_dep_info): platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice2', description='test platform device') platform_device_id = self.imsclient.create_platform_device(platform_device_obj) instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice2', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.imsclient.assign_platform_model_to_platform_device(prior_dep_info.platform_model_id, platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device(prior_dep_info.instrument_model_id, instrument_device_id) start = str(int(time.mktime(datetime.datetime(2013, 6, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2020, 6, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment2', description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment(prior_dep_info.platform_site_id, deployment_id) self.omsclient.assign_device_to_deployment(prior_dep_info.platform_device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) ret = DotDict(instrument_device_id=instrument_device_id, platform_device_id=platform_device_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment(make_assigns=True) before_activate_instrument_device_obj = self.rrclient.read(res.instrument_device_id) self.assertNotEquals(before_activate_instrument_device_obj.lcstate, LCS.DEPLOYED) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) # OOIION-1239: retrieve the extended resource and validate that only two sites are in the list of portals extended_deployment = self.omsclient.get_deployment_extension(res.deployment_id) self.assertEquals( len(extended_deployment.computed.portals.value), 2) def assertGeospatialBoundsEquals(a, b): self.assertEquals(a['geospatial_latitude_limit_north'],b['geospatial_latitude_limit_north']) self.assertEquals(a['geospatial_latitude_limit_south'],b['geospatial_latitude_limit_south']) self.assertEquals(a['geospatial_longitude_limit_west'],b['geospatial_longitude_limit_west']) self.assertEquals(a['geospatial_longitude_limit_east'],b['geospatial_longitude_limit_east']) def assertGeospatialBoundsNotEquals(a, b): self.assertNotEquals(a['geospatial_latitude_limit_north'],b['geospatial_latitude_limit_north']) self.assertNotEquals(a['geospatial_latitude_limit_south'],b['geospatial_latitude_limit_south']) self.assertNotEquals(a['geospatial_longitude_limit_west'],b['geospatial_longitude_limit_west']) self.assertNotEquals(a['geospatial_longitude_limit_east'],b['geospatial_longitude_limit_east']) after_activate_instrument_device_obj = self.rrclient.read(res.instrument_device_id) assertGeospatialBoundsNotEquals(before_activate_instrument_device_obj.geospatial_bounds,after_activate_instrument_device_obj.geospatial_bounds) deployment_obj = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj.lcstate, LCS.DEPLOYED) extended_deployment = self.omsclient.get_deployment_extension(res.deployment_id) # two sites in this test self.assertEquals(len(extended_deployment.computed.portals.value), 2) # only one portal instrument self.assertEquals(len(extended_deployment.portal_instruments), 1) log.debug("deactivatin deployment, expecting success") self.omsclient.deactivate_deployment(res.deployment_id) after_deactivate_instrument_device_obj = self.rrclient.read(res.instrument_device_id) assertGeospatialBoundsNotEquals(after_activate_instrument_device_obj.geospatial_bounds, after_deactivate_instrument_device_obj.geospatial_bounds) deployment_obj = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj.lcstate, LCS.INTEGRATED) def test_activate_deployment_redeploy(self): dep_util = DeploymentUtil(self.container) res = self.base_activate_deployment(make_assigns=True) log.debug("activating first deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) deployment_obj1 = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj1.lcstate, LCS.DEPLOYED) next_dep_info = self._create_subsequent_deployment(res) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertNotEquals(deployment_obj2.lcstate, LCS.DEPLOYED) log.debug("activating subsequent deployment, expecting success") self.omsclient.activate_deployment(next_dep_info.deployment_id) deployment_obj1 = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj1.lcstate, LCS.INTEGRATED) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertEquals(deployment_obj2.lcstate, LCS.DEPLOYED) dep1_tc = dep_util.get_temporal_constraint(deployment_obj1) dep2_tc = dep_util.get_temporal_constraint(deployment_obj2) self.assertLessEqual(float(dep1_tc.end_datetime), float(dep2_tc.end_datetime)) log.debug("deactivating second deployment, expecting success") self.omsclient.deactivate_deployment(next_dep_info.deployment_id) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertEquals(deployment_obj2.lcstate, LCS.INTEGRATED) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.assign_site_to_deployment(res.platform_site_id, res.deployment_id) self.omsclient.assign_device_to_deployment(res.platform_device_id, res.deployment_id) log.debug("activating deployment without site+device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.omsclient.assign_device_to_deployment(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without instrument site, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest) #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.assign_site_to_deployment(res.instrument_site_id, res.deployment_id) log.debug("activating deployment without device, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment") def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message=""): with self.assertRaises(err_type) as cm: self.omsclient.activate_deployment(deployment_id) log.debug("assert_deploy_fail cm: %s", str(cm) ) if fail_message: self.assertIn(fail_message, cm.exception.message) def test_3x3_matchups_remoteplatform(self): self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext)) def test_3x3_matchups_cabledinstrument(self): self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext)) def test_3x3_matchups_cablednode(self): self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext)) def base_3x3_matchups(self, deployment_context): """ This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1) """ deployment_context_type = type(deployment_context).__name__ instrument_model_id = [self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)] platform_model_id = [self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)] instrument_device_id = [self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)] platform_device_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)] instrument_site_id = [self.RR2.create(any_old(RT.InstrumentSite, { "reference_designator" : "GA01SUMO-FI003-0%s-CTDMO0999" % (i+1), "planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="GA01SUMO-FI003-0%s-CTDMO0999" % (i+1) )})) for i in range(9)] platform_site_id = [self.RR2.create(any_old(RT.PlatformSite, { "reference_designator" : "GA01SUMO-FI003-0%s-CTDMO0888" % (i+1) , "planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="GA01SUMO-FI003-0%s-CTDMO0888" % (i+1))})) for i in range(4)] def instrument_model_at(platform_idx, instrument_idx): m = platform_idx * 2 if instrument_idx > 0: m += 1 return m def platform_model_at(platform_idx): if platform_idx > 0: return 1 return 0 def instrument_at(platform_idx, instrument_idx): return platform_idx * 3 + instrument_idx # set up the structure for p in range(3): m = platform_model_at(p) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[m], platform_site_id[p]) self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[m], platform_device_id[p]) self.RR2.assign_platform_device_to_platform_device_with_has_device(platform_device_id[p], platform_device_id[3]) self.RR2.assign_platform_site_to_platform_site_with_has_site(platform_site_id[p], platform_site_id[3]) for i in range(3): m = instrument_model_at(p, i) idx = instrument_at(p, i) self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id[m], instrument_site_id[idx]) self.RR2.assign_instrument_model_to_instrument_device_with_has_model(instrument_model_id[m], instrument_device_id[idx]) self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id[idx], platform_device_id[p]) self.RR2.assign_instrument_site_to_platform_site_with_has_site(instrument_site_id[idx], platform_site_id[p]) # top level models self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[2], platform_device_id[3]) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[2], platform_site_id[3]) # verify structure for p in range(3): parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(platform_device_id[p]) self.assertEqual(platform_device_id[3], parent_id) parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(platform_site_id[p]) self.assertEqual(platform_site_id[3], parent_id) for i in range(len(platform_site_id)): self.assertEqual(self.RR2.find_platform_model_of_platform_device_using_has_model(platform_device_id[i]), self.RR2.find_platform_model_of_platform_site_using_has_model(platform_site_id[i])) for i in range(len(instrument_site_id)): self.assertEqual(self.RR2.find_instrument_model_of_instrument_device_using_has_model(instrument_device_id[i]), self.RR2.find_instrument_model_of_instrument_site_using_has_model(instrument_site_id[i])) # OOIReferenceDesignator format: GA01SUMO-FI003-03-CTDMO0999 (site-platform_id-port-device_id) port_assignments = {} for p in range(3): ref_desig = "GA01SUMO-FI003-0%s-CTDMO0888" % (p+1) pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type= PortTypeEnum.PAYLOAD, ip_address=str(p) ) port_assignments[platform_device_id[p]] = pp_obj for i in range(3): ref_desig = "GA01SUMO-FI003-0%s-CTDMO0999" % ((p*3)+i+1) pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type= PortTypeEnum.PAYLOAD, ip_address=str(p) ) idx = instrument_at(p, i) port_assignments[instrument_device_id[idx]] = pp_obj deployment_id = self.RR2.create(any_old(RT.Deployment, {"context": deployment_context, "port_assignments": port_assignments})) log.debug("assigning device/site to %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[3]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[3]) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_instrument_device_with_has_deployment(deployment_id, instrument_device_id[1]) self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id[1]) elif OT.CabledNodeDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[1]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[1]) log.debug("activation of %s deployment", deployment_context_type) self.omsclient.activate_deployment(deployment_id) log.debug("validation of %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(d, self.RR2.find_platform_device_id_of_platform_site_using_has_device(platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual(d, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[i])) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.assertEqual(instrument_device_id[1], self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[1])) elif OT.CabledNodeDeploymentContext == deployment_context_type: expected_platforms = [1] # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(i in expected_platforms, d in self.RR2.find_platform_device_ids_of_platform_site_using_has_device(platform_site_id[i]))
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() # self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.container.start_rel_from_url("res/deploy/r2deploy.yml") print "started services" # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) def cleanupprocs(self): stm = os.popen("ps -e | grep ion.agents.port.logger_process") procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) stm = os.popen("ps -e | grep ion.agents.instrument.zmq_driver_process") procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) stm = os.popen("rm /tmp/*.pid.txt") @unittest.skip("timeout on start inst agent?") def test_reassignPrimaryDevice(self): # ensure no processes or pids are left around by agents or Sims self.cleanupprocs() # Set up the preconditions # Set up the preconditions # ingestion configuration parameters self.exchange_point_id = "science_data" self.number_of_workers = 2 self.hdf_storage = HdfStorage(relative_path="ingest") self.couch_storage = CouchStorage(datastore_name="test_datastore") self.XP = "science_data" self.exchange_name = "ingestion_queue" # ------------------------------- # Create ingestion configuration and activate it # ------------------------------- ingestion_configuration_id = self.ingestclient.create_ingestion_configuration( exchange_point_id=self.exchange_point_id, couch_storage=self.couch_storage, hdf_storage=self.hdf_storage, number_of_workers=self.number_of_workers, ) print "test_deployAsPrimaryDevice: ingestion_configuration_id", ingestion_configuration_id # activate an ingestion configuration ret = self.ingestclient.activate_ingestion_configuration(ingestion_configuration_id) # ------------------------------- # Create InstrumentModel # ------------------------------- instModel_obj = IonObject( RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model_label="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) print "test_deployAsPrimaryDevice: new InstrumentModel id = ", instModel_id # ------------------------------- # Create InstrumentAgent # ------------------------------- instAgent_obj = IonObject( RT.InstrumentAgent, name="agent007", description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent", ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) print "test_deployAsPrimaryDevice: new InstrumentAgent id = ", instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # ------------------------------- # Create Instrument Site # ------------------------------- instrumentSite_obj = IonObject(RT.InstrumentSite, name="instrumentSite1", description="SBE37IMInstrumentSite") try: instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id="") except BadRequest as ex: self.fail("failed to create new InstrumentSite: %s" % ex) print "test_deployAsPrimaryDevice: new instrumentSite id = ", instrumentSite_id self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id) # ------------------------------- # Create Old InstrumentDevice # ------------------------------- instDevice_obj = IonObject( RT.InstrumentDevice, name="SBE37IMDeviceYear1", description="SBE37IMDevice for the FIRST year of deployment", serial_number="12345", ) try: oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print "test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ", oldInstDevice_id # deploy this device to the logical slot self.omsclient.deploy_instrument_device_to_instrument_site(oldInstDevice_id, instrumentSite_id) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE) # set this device as the current primary device self.omsclient.deploy_as_primary_instrument_device_to_instrument_site(oldInstDevice_id, instrumentSite_id) # ------------------------------- # Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information # cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, # ------------------------------- instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstanceYear1", description="SBE37IMAgentInstance Year 1", svr_addr="localhost", driver_module="ion.agents.instrument.drivers.sbe37.sbe37_driver", driver_class="SBE37Driver", cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address="localhost", comms_device_port=4001, comms_server_address="localhost", comms_server_port=8888, ) oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, oldInstDevice_id ) # ------------------------------- # Create CTD Parsed as the Year 1 data product # ------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) print "test_deployAsPrimaryDevice: new Stream Definition id = ", ctd_stream_def_id print "Creating new CDM data product with a stream definition" dp_obj = IonObject(RT.DataProduct, name="ctd_parsed_year1", description="ctd stream test year 1") try: ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year1 self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1 ) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True) print "test_deployAsPrimaryDevice: Data product streams1 = ", stream_ids # ------------------------------- # Create New InstrumentDevice # ------------------------------- instDevice_obj_2 = IonObject( RT.InstrumentDevice, name="SBE37IMDeviceYear2", description="SBE37IMDevice for the SECOND year of deployment", serial_number="67890", ) try: newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print "test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ", newInstDevice_id # deploy this device to the logical slot self.omsclient.deploy_instrument_device_to_instrument_site(newInstDevice_id, instrumentSite_id) # set the LCSTATE self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE) instDevice_obj_2 = self.rrclient.read(newInstDevice_id) log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate)) # ------------------------------- # Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information # ------------------------------- instAgentInstance_new__obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstanceYear2", description="SBE37IMAgentInstance Year 2", svr_addr="localhost", driver_module="ion.agents.instrument.drivers.sbe37.sbe37_driver", driver_class="SBE37Driver", cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address="localhost", comms_device_port=4002, comms_server_address="localhost", comms_server_port=8888, ) newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_new__obj, instAgent_id, newInstDevice_id ) # ------------------------------- # Create CTD Parsed as the Year 2 data product # ------------------------------- # create a stream definition for the data from the ctd simulator # ctd_stream_def = SBE37_CDM_stream_definition() # ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) print "test_deployAsPrimaryDevice: new Stream Definition id = ", ctd_stream_def_id print "Creating new CDM data product with a stream definition" dp_obj = IonObject(RT.DataProduct, name="ctd_parsed_year2", description="ctd stream test year 2") try: ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year2 self.damsclient.assign_data_product( input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2 ) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True) print "test_deployAsPrimaryDevice: Data product streams2 = ", stream_ids # ------------------------------- # Logical Data Product: Data Process Definition # ------------------------------- # log.debug(" test_deployAsPrimaryDevice: create data process definition logical_transform") # dpd_obj = IonObject(RT.DataProcessDefinition, # name='logical_transform', # description='send the packet from the in stream to the out stream unchanged', # module='ion.processes.data.transforms.logical_transform', # class_name='logical_transform', # process_source='some_source_reference') # try: # logical_transform_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # except BadRequest as ex: # self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition # ------------------------------- log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all") dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L0_all", description="transform ctd package into three separate L0 streams", module="ion.processes.data.transforms.ctd.ctd_L0_all", class_name="ctd_L0_all", process_source="some_source_reference", ) try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" % ex) # ------------------------------- # Logical Transform: Output Data Products # ------------------------------- # outgoing_logical_stream_def = SBE37_CDM_stream_definition() # outgoing_logical_stream_def_id = self.pubsubclient.create_stream_definition(container=outgoing_logical_stream_def) # self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_logical_stream_def_id, logical_transform_dprocdef_id ) # # log.debug("test_deployAsPrimaryDevice: create output parsed data product for Logical Instrument") # ctd_logical_output_dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_logical',description='ctd parsed from the logical instrument') # instrument_site_output_dp_id = self.dataproductclient.create_data_product(ctd_logical_output_dp_obj, outgoing_logical_stream_def_id) # self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id, persist_data=True, persist_metadata=True) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products # ------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l0_conductivity, name="L0_Conductivity" ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l0_pressure, name="L0_Pressure" ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l0_temperature, name="L0_Temperature" ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products = {} log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name="L0_Conductivity", description="transform output conductivity" ) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id ) self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True) log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name="L0_Pressure", description="transform output pressure" ) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id ) self.output_products["pressure"] = ctd_l0_pressure_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True) log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name="L0_Temperature", description="transform output temperature" ) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id ) self.output_products["temperature"] = ctd_l0_temperature_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True) # ------------------------------- # CTD Logical: Create the data process # ------------------------------- # log.debug("test_deployAsPrimaryDevice: create ctd_parsed logical data_process start") # try: # ctd_parsed_logical_data_process_id = self.dataprocessclient.create_data_process(logical_transform_dprocdef_id, ctd_parsed_data_product_year1, {'output':instrument_site_output_dp_id}) # self.dataprocessclient.activate_data_process(ctd_parsed_logical_data_process_id) # except BadRequest as ex: # self.fail("failed to create new data process: %s" %ex) # log.debug("test_deployAsPrimaryDevice: create L0 all data_process return") # ------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product) # ------------------------------- log.debug("test_deployAsPrimaryDevice: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products ) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug("test_deployAsPrimaryDevice: create L0 all data_process return") # ------------------------------- # Launch InstrumentAgentInstance Sim1, connect to the resource agent client # ------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id) inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent1_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim1 = ResourceAgentClient( "iaclient Sim1", name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess() ) print "activate_instrument: got _ia_client_sim1 %s", self._ia_client_sim1 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1)) # ------------------------------- # Launch InstrumentAgentInstance Sim2, connect to the resource agent client # ------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id) inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent2_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim2 = ResourceAgentClient( "iaclient Sim2", name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess() ) print "activate_instrument: got _ia_client_sim2 %s", self._ia_client_sim2 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2)) # ------------------------------- # Streaming Sim1 (old instrument) # ------------------------------- cmd = AgentCommand(command="initialize") retval = self._ia_client_sim1.execute_agent(cmd) print retval log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 initialize %s", str(retval)) time.sleep(2) cmd = AgentCommand(command="go_active") reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command="run") reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 run %s", str(reply)) time.sleep(2) log.debug("test_activateInstrument: calling go_streaming ") cmd = AgentCommand(command="go_streaming") reply = self._ia_client_sim1.execute(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_streaming %s", str(reply)) # ------------------------------- # Streaming Sim 2 (new instrument) # ------------------------------- cmd = AgentCommand(command="initialize") retval = self._ia_client_sim2.execute_agent(cmd) print retval log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 initialize %s", str(retval)) time.sleep(2) cmd = AgentCommand(command="go_active") reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command="run") reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 run %s", str(reply)) time.sleep(2) log.debug("test_activateInstrument: calling go_streaming ") cmd = AgentCommand(command="go_streaming") reply = self._ia_client_sim2.execute(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_streaming %s", str(reply)) time.sleep(20) # ------------------------------- # Shutdown Sim1 (old instrument) # ------------------------------- log.debug("test_activateInstrument: calling go_observatory") cmd = AgentCommand(command="go_observatory") reply = self._ia_client_sim1.execute(cmd) log.debug("test_activateInstrument: _ia_client_sim1 return from go_observatory %s", str(reply)) time.sleep(5) log.debug("test_deployAsPrimaryDevice:: calling go_inactive ") cmd = AgentCommand(command="go_inactive") reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from go_inactive %s", str(reply)) time.sleep(2) log.debug("test_deployAsPrimaryDevice:: calling reset ") cmd = AgentCommand(command="reset") reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply)) time.sleep(2) # ------------------------------- # Shutdown Sim2 (old instrument) # ------------------------------- log.debug("test_activateInstrument: calling go_observatory") cmd = AgentCommand(command="go_observatory") reply = self._ia_client_sim2.execute(cmd) log.debug("test_activateInstrument: _ia_client_sim2 return from go_observatory %s", str(reply)) time.sleep(8) log.debug("test_deployAsPrimaryDevice:: calling go_inactive ") cmd = AgentCommand(command="go_inactive") reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from go_inactive %s", str(reply)) time.sleep(2) log.debug("test_deployAsPrimaryDevice:: calling reset ") cmd = AgentCommand(command="reset") reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply)) time.sleep(2) log.debug("test_deployAsPrimaryDevice:: calling reset ") cmd = AgentCommand(command="reset") reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from reset %s", str(reply)) time.sleep(2) self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id) self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment( deployment_obj, site_id, device_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj)) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.omsclient.delete_deployment(deployment_id) # now try to get the deleted dp object try: deployment_obj = self.omsclient.read_deployment(deployment_id) except NotFound as ex: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_activate_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') model_id = self.imsclient.create_platform_model(platform_model__obj) self.imsclient.assign_platform_model_to_platform_device( model_id, device_id) self.omsclient.assign_platform_model_to_platform_site( model_id, site_id) #create a deployment with metadata and an initial site and device instrument_site__obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site( instrument_site__obj, site_id) instrument_device__obj = IonObject( RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device__obj) self.rrclient.create_association(device_id, PRED.hasDevice, instrument_device_id) instrument_model__obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model( instrument_model__obj) self.imsclient.assign_instrument_model_to_instrument_device( instrument_model_id, instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( instrument_model_id, instrument_site_id) #self.rrclient.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment( deployment_obj, site_id, device_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) self.omsclient.activate_deployment(deployment_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() #self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) def cleanupprocs(self): stm = os.popen('ps -e | grep ion.agents.port.logger_process') procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid,signal.SIGKILL) stm = os.popen('ps -e | grep ion.agents.instrument.zmq_driver_process') procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid,signal.SIGKILL) # stm = os.popen('rm /tmp/*.pid.txt') @unittest.skip ("Deprecated by IngestionManagement refactor, timeout on start inst agent?") def test_deploy_activate_full(self): # ensure no processes or pids are left around by agents or Sims self.cleanupprocs() # Set up the preconditions # Set up the preconditions # ingestion configuration parameters exchange_point_id = 'science_data' exchange_name = 'ingestion_queue' #------------------------------- # Create ingestion configuration and activate it #------------------------------- q = IngestionQueue() ingestion_configuration_id = self.ingestclient.create_ingestion_configuration( name=exchange_name, exchange_point_id=exchange_point_id, queues=[q]) print 'test_deployAsPrimaryDevice: ingestion_configuration_id', ingestion_configuration_id # activate an ingestion configuration # = self.ingestclient.activate_ingestion_configuration(ingestion_configuration_id) #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) print 'test_deployAsPrimaryDevice: new InstrumentModel id = ', instModel_id #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) print 'test_deployAsPrimaryDevice: new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create Instrument Site #------------------------------- instrumentSite_obj = IonObject(RT.InstrumentSite, name='instrumentSite1', description="SBE37IMInstrumentSite" ) try: instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id='') except BadRequest as ex: self.fail("failed to create new InstrumentSite: %s" %ex) print 'test_deployAsPrimaryDevice: new instrumentSite id = ', instrumentSite_id self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id) #------------------------------- # Logical Transform: Output Data Products #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) log.debug("test_deployAsPrimaryDevice: create output parsed data product for Logical Instrument") craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() ctd_logical_output_dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_logical', description='ctd parsed from the logical instrument', temporal_domain = tdom, spatial_domain = sdom) instrument_site_output_dp_id = self.dataproductclient.create_data_product(ctd_logical_output_dp_obj, ctd_stream_def_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id) self.omsclient.create_site_data_product(instrumentSite_id, instrument_site_output_dp_id) #------------------------------- # Create Old InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDeviceYear1', description="SBE37IMDevice for the FIRST year of deployment", serial_number="12345" ) try: oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) print 'test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ', oldInstDevice_id self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='first deployment') oldDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id) #------------------------------- # Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information # cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, #------------------------------- instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear1', description="SBE37IMAgentInstance Year 1", svr_addr="localhost", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address="localhost", comms_device_port=4001, comms_server_address="localhost", comms_server_port=8888) oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, oldInstDevice_id) #------------------------------- # Create CTD Parsed as the Year 1 data product and attach to instrument #------------------------------- print 'test_deployAsPrimaryDevice: new Stream Definition id = ', ctd_stream_def_id print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year1', description='ctd stream test year 1', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year1 self.damsclient.assign_data_product(input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams1 = ', stream_ids #------------------------------- # Create New InstrumentDevice #------------------------------- instDevice_obj_2 = IonObject(RT.InstrumentDevice, name='SBE37IMDeviceYear2', description="SBE37IMDevice for the SECOND year of deployment", serial_number="67890" ) try: newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) print 'test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ', newInstDevice_id #set the LCSTATE self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE) instDevice_obj_2 = self.rrclient.read(newInstDevice_id) log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate)) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='second deployment') newDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id) #------------------------------- # Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information #------------------------------- instAgentInstance_new__obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear2', description="SBE37IMAgentInstance Year 2", svr_addr="localhost", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address="localhost", comms_device_port=4002, comms_server_address="localhost", comms_server_port=8888) newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_new__obj, instAgent_id, newInstDevice_id) #------------------------------- # Create CTD Parsed as the Year 2 data product #------------------------------- # create a stream definition for the data from the ctd simulator # ctd_stream_def = SBE37_CDM_stream_definition() # ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) print 'test_deployAsPrimaryDevice: new Stream Definition id = ', ctd_stream_def_id print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year2', description='ctd stream test year 2', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year2 self.damsclient.assign_data_product(input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products={} log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id, parameter_dictionary) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary ) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product) #------------------------------- log.debug("test_deployAsPrimaryDevice: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_deployAsPrimaryDevice: create L0 all data_process return") #-------------------------------- # Activate the deployment #-------------------------------- self.omsclient.activate_deployment(oldDeployment_id) #------------------------------- # Launch InstrumentAgentInstance Sim1, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id) inst_agent1_instance_obj= self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent1_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim1 = ResourceAgentClient('iaclient Sim1', name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim1 %s', self._ia_client_sim1 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1)) #------------------------------- # Launch InstrumentAgentInstance Sim2, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id) inst_agent2_instance_obj= self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent2_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim2 = ResourceAgentClient('iaclient Sim2', name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim2 %s', self._ia_client_sim2 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2)) #------------------------------- # Streaming Sim1 (old instrument) #------------------------------- cmd = AgentCommand(command='initialize') retval = self._ia_client_sim1.execute_agent(cmd) print retval log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 initialize %s", str(retval)) time.sleep(2) cmd = AgentCommand(command='go_active') reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command='run') reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 run %s", str(reply)) time.sleep(2) log.debug("test_activateInstrument: calling go_streaming ") cmd = AgentCommand(command='go_streaming') reply = self._ia_client_sim1.execute(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_streaming %s", str(reply)) #------------------------------- # Streaming Sim 2 (new instrument) #------------------------------- cmd = AgentCommand(command='initialize') retval = self._ia_client_sim2.execute_agent(cmd) print retval log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 initialize %s", str(retval)) time.sleep(2) cmd = AgentCommand(command='go_active') reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command='run') reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 run %s", str(reply)) time.sleep(2) log.debug("test_activateInstrument: calling go_streaming ") cmd = AgentCommand(command='go_streaming') reply = self._ia_client_sim2.execute(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_streaming %s", str(reply)) time.sleep(20) #------------------------------- # Shutdown Sim1 (old instrument) #------------------------------- log.debug("test_activateInstrument: calling go_observatory") cmd = AgentCommand(command='go_observatory') reply = self._ia_client_sim1.execute(cmd) log.debug("test_activateInstrument: _ia_client_sim1 return from go_observatory %s", str(reply)) time.sleep(5) log.debug("test_deployAsPrimaryDevice:: calling go_inactive ") cmd = AgentCommand(command='go_inactive') reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from go_inactive %s", str(reply)) time.sleep(2) log.debug("test_deployAsPrimaryDevice:: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply)) time.sleep(2) #------------------------------- # Shutdown Sim2 (old instrument) #------------------------------- log.debug("test_activateInstrument: calling go_observatory") cmd = AgentCommand(command='go_observatory') reply = self._ia_client_sim2.execute(cmd) log.debug("test_activateInstrument: _ia_client_sim2 return from go_observatory %s", str(reply)) time.sleep(8) log.debug("test_deployAsPrimaryDevice:: calling go_inactive ") cmd = AgentCommand(command='go_inactive') reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from go_inactive %s", str(reply)) time.sleep(2) log.debug("test_deployAsPrimaryDevice:: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply)) time.sleep(2) log.debug("test_deployAsPrimaryDevice:: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from reset %s", str(reply)) time.sleep(2) self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id) self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url("res/deploy/r2deploy.yml") self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dmpsclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.psmsclient = PubsubManagementServiceClient(node=self.container.node) # @unittest.skip("targeting") def test_create_deployment(self): # create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name="PlatformSite1", description="test platform site") site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name="PlatformDevice1", description="test platform device") device_id = self.imsclient.create_platform_device(platform_device__obj) deployment_obj = IonObject(RT.Deployment, name="TestDeployment", description="some new deployment") deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) # retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj)) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) # delete the deployment self.omsclient.delete_deployment(deployment_id) # now try to get the deleted dp object try: deployment_obj = self.omsclient.read_deployment(deployment_id) except NotFound as ex: pass else: self.fail("deleted deployment was found during read") # @unittest.skip("targeting") def test_activate_deployment(self): # create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name="PlatformSite1", description="test platform site") site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name="PlatformDevice1", description="test platform device") platform_device_id = self.imsclient.create_platform_device(platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name="PlatformModel1", description="test platform model") model_id = self.imsclient.create_platform_model(platform_model__obj) self.imsclient.assign_platform_model_to_platform_device(model_id, platform_device_id) self.omsclient.assign_platform_model_to_platform_site(model_id, site_id) # create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name="InstrumentSite1", description="test instrument site") instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, site_id) # assign data products appropriately # set up stream (this would be preload) ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.psmsclient.create_stream_definition(container=ctd_stream_def) craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject( RT.DataProduct, name="DP1", description="some new dp", temporal_domain=tdom, spatial_domain=sdom ) log_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id) instrument_device_obj = IonObject( RT.InstrumentDevice, name="InstrumentDevice1", description="test instrument device" ) instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) dp_obj = IonObject( RT.DataProduct, name="DP1", description="some new dp", temporal_domain=tdom, spatial_domain=sdom ) inst_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) # assign data products appropriately self.damsclient.assign_data_product( input_resource_id=instrument_device_id, data_product_id=inst_data_product_id ) instrument_model_obj = IonObject( RT.InstrumentModel, name="InstrumentModel1", description="test instrument model" ) instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj) self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(instrument_model_id, instrument_site_id) # self.rrclient.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) deployment_obj = IonObject(RT.Deployment, name="TestDeployment", description="some new deployment") deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instrument_device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) self.omsclient.activate_deployment(deployment_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj, site_id, device_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) ) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.omsclient.delete_deployment(deployment_id) # now try to get the deleted dp object try: deployment_obj = self.omsclient.read_deployment(deployment_id) except NotFound as ex: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_activate_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') model_id = self.imsclient.create_platform_model(platform_model__obj) self.imsclient.assign_platform_model_to_platform_device(model_id, device_id) self.omsclient.assign_platform_model_to_platform_site(model_id, site_id) #create a deployment with metadata and an initial site and device instrument_site__obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site__obj, site_id) instrument_device__obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device__obj) self.rrclient.create_association(device_id, PRED.hasDevice, instrument_device_id) instrument_model__obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model(instrument_model__obj) self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(instrument_model_id, instrument_site_id) #self.rrclient.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj, site_id, device_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dmpsclient = DataProductManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.psmsclient = PubsubManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) # create missing data process definition self.dsmsclient = DataProcessManagementServiceClient( node=self.container.node) dpd_obj = IonObject( RT.DataProcessDefinition, name=LOGICAL_TRANSFORM_DEFINITION_NAME, description="normally in preload", module='ion.processes.data.transforms.logical_transform', class_name='logical_transform') self.dsmsclient.create_data_process_definition(dpd_obj) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources( RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013, 1, 1)) end = IonTime(datetime.datetime(2014, 1, 1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj)) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.RR2.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self): #------------------------------------------------------------------------------------- # Create platform site, platform device, platform model #------------------------------------------------------------------------------------- platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') platform_site_id = self.omsclient.create_platform_site( platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device( platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model( platform_model__obj) #------------------------------------------------------------------------------------- # Create instrument site #------------------------------------------------------------------------------------- instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site( instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='Log Data Product', description='some new dp', temporal_domain=tdom, spatial_domain=sdom) out_log_data_product_id = self.dmpsclient.create_data_product( dp_obj, ctd_stream_def_id) #---------------------------------------------------------------------------------------------------- # Start the transform (a logical transform) that acts as an instrument site #---------------------------------------------------------------------------------------------------- self.omsclient.create_site_data_product( site_id=instrument_site_id, data_product_id=out_log_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument device #---------------------------------------------------------------------------------------------------- instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) dp_obj = IonObject(RT.DataProduct, name='Instrument Data Product', description='some new dp', temporal_domain=tdom, spatial_domain=sdom) inst_data_product_id = self.dmpsclient.create_data_product( dp_obj, ctd_stream_def_id) #assign data products appropriately self.damsclient.assign_data_product( input_resource_id=instrument_device_id, data_product_id=inst_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument model #---------------------------------------------------------------------------------------------------- instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model( instrument_model_obj) #---------------------------------------------------------------------------------------------------- # Create a deployment object #---------------------------------------------------------------------------------------------------- start = IonTime(datetime.datetime(2013, 1, 1)) end = IonTime(datetime.datetime(2014, 1, 1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_platform_model_to_platform_device( res.platform_model_id, res.platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_platform_model_to_platform_site( res.platform_model_id, res.platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("adding instrument site and device to deployment") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("adding platform site and device to deployment") self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id) self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug( "activating deployment without site+device models, expecting fail") self.assert_deploy_fail( res.deployment_id, "Expected at least 1 model for InstrumentSite") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug( "activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, "Expected 1 model for InstrumentDevice") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug( "activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, "No sites were found in the deployment") #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) log.debug( "activating deployment without device models, expecting fail") self.assert_deploy_fail( res.deployment_id, "The set of devices could not be mapped to the set of sites") def assert_deploy_fail(self, deployment_id, fail_message="did not specify fail_message"): with self.assertRaises(BadRequest) as cm: self.omsclient.activate_deployment(deployment_id) self.assertIn(fail_message, cm.exception.message)
class TestRegisterAndActivate(IonIntegrationTestCase): """ Integration test cases to confirm registration and activation services for marine device resources. """ def setUp(self): """ Test setup. """ # Resources used in the tests. # General resources. self.actor_id = None self.user_info_id = None self.org_id = None self.obs_id = None # Cabled infrastructure. self.cabled_platform_model_id = None self.cabled_platform_site_id = None self.cabled_platform_device_id = None self.cabled_platform_agent_id = None self.cabled_platform_agent_instance_id = None self.cabled_platform_deployment_id = None self.cabled_instrument_deployment_id = None self.cabled_instrument_model_id = None self.cabled_instrument_site_id = None self.cabled_instrument_device_id = None self.cabled_instrument_agent_id = None self.cabled_instrument_agent_instance_id = None self.cabled_instrument_deployment_id = None # Uncabled infrastructure. self.uncabled_platform_model_id = None self.uncabled_platform_site_id = None self.uncabled_platform_device_id = None self.uncabled_platform_agent_id = None self.uncabled_platform_agent_instance_id = None self.uncabled_instrument_model_id = None self.uncabled_instrument_site_id = None self.uncabled_instrument_device_id = None self.uncabled_instrument_agent_id = None self.uncabled_instrument_agent_instance_id = None self.uncabled_site_deployment_id = None # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Setup service clients. self.idms = IdentityManagementServiceClient(node=self.container.node) self.oms = ObservatoryManagementServiceClient(node=self.container.node) # Add generic resources. self._load_system_actors() self._create_user() self._create_org() self._create_observatory() # Add cleanup routine. self.addCleanup(self._cleanup_resources) def _cleanup_resources(self): """ Delete resources created by the tests. """ # Check and clean up cabled resources. if self.cabled_instrument_model_id: self.oms.unassign_instrument_model_from_instrument_site( self.cabled_instrument_model_id, self.cabled_instrument_site_id) self.ims.delete_instrument_model(self.cabled_instrument_model_id) self.cabled_instrument_model_id = None if self.cabled_platform_model_id: self.oms.unassign_platform_model_from_platform_site( self.cabled_platform_model_id, self.cabled_platform_site_id) self.ims.delete_platform_model(self.cabled_platform_model_id) self.cabled_platform_model_id = None if self.cabled_instrument_site_id: self.oms.unassign_site_from_site(self.cabled_instrument_site_id, self.cabled_platform_site_id) self.oms.delete_instrument_site(self.cabled_instrument_site_id) self.cabled_instrument_site_id = None if self.cabled_platform_site_id: self.oms.unassign_site_from_site(self.cabled_platform_site_id, self.obs_id) self.oms.delete_platform_site(self.cabled_platform_site_id) self.cabled_platform_site_id = None # Check and clean up unclabled resources. # TODO # Clean up generic resources. if self.user_info_id: self.idms.delete_user_info(self.user_info_id) self.user_info_id = None if self.actor_id: self.idms.delete_actor_identity(self.actor_id) self.actor_id = None if self.obs_id: self.oms.delete_observatory(self.obs_id) self.obs_id = None if self.org_id: self.container.resource_registry.delete(self.org_id) self.org_id = None def _load_system_actors(self): """ Retrieve system and webauth actors and headers for later use. """ # Retrieve and store system actor and headers. system_actor, _ = self.container.resource_registry.find_resources( RT.ActorIdentity, name=CFG.system.system_actor, id_only=False) self.system_actor = system_actor[0] if system_actor else None self.system_actor_id = system_actor[0]._id if system_actor \ else 'anonymous' self.system_actor_headers = { 'ion-actor-id': self.system_actor_id, 'ion-actor-roles': {'ION': ['ION_MANAGER', 'ORG_MANAGER']}, 'expiry':'0' } # Retrieve and store webauth actor and headers. webauth_actor, _ = self.container.resource_registry.find_resources( RT.ActorIdentity, name=CFG.get_safe("system.web_authentication_actor", "web_authentication"), id_only=False) self.webauth_actor = webauth_actor[0] if webauth_actor else None self.webauth_actor_id = webauth_actor[0]._id if webauth_actor \ else 'anonymous' self.webauth_actor_headers = { 'ion-actor-id': self.webauth_actor_id, 'ion-actor-roles': {'ION': ['ION_MANAGER', 'ORG_MANAGER']}, 'expiry':'0' } def _create_user(self): """ Create user resources that serve as device owners. This test user does not have contact information, user credentials or notification preferences. Results in these objects: ActorIdentity({'_rev': '1', '_id': '07f92986b34e426bba0fca00b73cf4a5', 'lcstate': 'DEPLOYED', 'alt_ids': [], 'description': '', 'ts_updated': '1391542388312', 'actor_type': 1, 'addl': {}, 'ts_created': '1391542388312', 'availability': 'AVAILABLE', 'name': 'Identity for Adam Activationtest'}) UserInfo({'_rev': '1', '_id': 'ac8d368e6ea247d996fd60dd0f9c7f89', 'lcstate': 'DEPLOYED', 'alt_ids': [], 'description': 'Activation Test User', 'tokens': [], 'ts_updated': '1391542388345', 'contact': ContactInformation({'individual_names_given': '', 'city': '', 'roles': [], 'administrative_area': '', 'url': '', 'country': '', 'variables': [{'name': '', 'value': ''}], 'organization_name': '', 'postal_code': '', 'individual_name_family': '', 'phones': [], 'position_name': '', 'email': '', 'street_address': ''}), 'variables': [{'name': '', 'value': ''}], 'addl': {}, 'ts_created': '1391542388345', 'availability': 'AVAILABLE', 'name': 'Adam Activationtest'}) """ # Basic user attributes for test user. user_attrs = { 'name' : 'Adam Activationtest', 'description' : 'Activation Test User' } # Create ActorIdentity. actor_name = "Identity for %s" % user_attrs['name'] actor_identity_obj = IonObject("ActorIdentity", name=actor_name) log.trace("creating user %s with headers: %r", user_attrs['name'], self.webauth_actor_headers) self.actor_id = self.idms.create_actor_identity(actor_identity_obj, headers=self.webauth_actor_headers) # Create UserInfo. user_info_obj = IonObject("UserInfo", **user_attrs) self.user_info_id = self.idms.create_user_info(self.actor_id, user_info_obj,headers=self.webauth_actor_headers) def _create_org(self): """ Create an org that contains all test infrastructure. Results in this object: Org({'message_controllable': True, '_rev': '1', '_id': '9ff82d9f6c7b41f886c6137f54a3086c', 'lcstate': 'DEPLOYED', 'alt_ids': [], 'url': '', 'description': 'An Org for Activation Tests', 'contacts': [], 'org_governance_name': 'ActiveOrg', 'institution': Institution({'website': '', 'phone': '', 'name': '', 'email': ''}), 'ts_updated': '1391542388395', 'monitorable': True, 'org_type': 2, 'addl': {}, 'ts_created': '1391542388395', 'availability': 'AVAILABLE', 'name': 'ActiveOrg'}) """ org_attrs = { 'name' : 'ActiveOrg', 'description' : 'An Org for Activation Tests', 'org_type' : OrgTypeEnum.MARINE_FACILITY } org_obj = IonObject('Org', **org_attrs) self.org_id = self.oms.create_marine_facility(org_obj, headers=self.system_actor_headers) def _create_observatory(self): """ Create a top level obsevaotry site for the tests. Results in this object: Observatory({'reference_designator': '', 'spatial_area_name': '', '_id': 'fdcda51901464575913858f98aaf0f41', '_rev': '1', 'lcstate': 'DEPLOYED', 'alt_ids': [], 'url': '', 'description': 'An Observatory for Activation Tests', 'coordinate_reference_system': GeospatialCoordinateReferenceSystem( {'geospatial_latitude_units': '', 'geospatial_vertical_crs': '', 'geospatial_geodetic_crs': '', 'geospatial_vertical_positive': '', 'geospatial_vertical_units': '', 'geospatial_longitude_units': ''}), 'constraint_list': [], 'environment': 1, 'ts_updated': '1391544601340', 'local_name': '', 'geospatial_point_center': GeospatialIndex({'lat': 0.0, 'lon': 0.0}), 'addl': {}, 'ts_created': '1391544601340', 'availability': 'AVAILABLE', 'name': 'ActiveObservatory'}) """ obs_attrs = { 'name': 'ActiveObservatory', 'description' : 'An Observatory for Activation Tests' } obs_obj = IonObject('Observatory', **obs_attrs) self.obs_id = self.oms.create_observatory(obs_obj, self.org_id) def _create_cabled_resources(self): """ Create preexisting infrastructure for the cabled test environment: sites, deployments, models. These are resources that already exist in the system due to preload or incremental preload updates. PlatformModel InstrumentModel PlatformSite InstrumentSite """ platform_model_attrs = { 'name' : 'LP Jbox', 'description' : 'Node Type: LJ', 'manufacturer' : 'University of Washington', 'platform_type' : 'Cable Node', 'platform_family' : 'Low Power JBox', 'ci_onboard' : False, 'shore_networked' : True } instrument_model_attrs = { 'name': 'Diffuse Vent Fluid 3-D Temperature Array (TMPSF-A)', 'description': 'Measures temperatures of diffuse flow across the seafloor', 'reference_designator': 'TMPSFA', 'class_name': 'Temperature seafloor', 'mixed_sampling_mode': True, 'integrated_inductive_modem_available': True, 'internal_battery': True, 'addl': {'comments': '', 'connector': '', 'makemodel_description': 'XR-420', 'input_voltage_range': '', 'interface': '', 'output_description': '', 'class_long_name': 'Temperature_seafloor'}, 'ooi_make_model': 'XR-420', 'series_name': 'TMPSF Series A', 'inline_management': True, 'series_id': 'TMPSFA', 'subseries_name': 'TMPSF Series A Sub 01', 'primary_interface': 1, 'manufacturer': 'RBR Global', 'family_name': 'Seafloor Properties', 'class_description': 'Measures temperatures of diffuse flow across the seafloor', 'class_alternate_name': 'Diffuse Vent Fluid 3-D Temperature Array', 'subseries_id': 'TMPSFA01', 'class_id': 'TMPSF', 'family_id': 'SFL', 'has_clock': True } platform_site_attrs = { 'name' : 'Cabled LP JBOX Platform Site', 'description' : 'Test Site for a Cabled LP JBOX Platform' } instrument_site_attrs = { 'name' : 'Cabled TMPSF Instrument Site', 'description' : 'Test Site for a Cabled TMPSF Instrument' } # Create the cabled model preloaded resources. platform_model = IonObject('PlatformModel', **platform_model_attrs) self.cabled_platform_model_id = self.ims.create_platform_model(platform_model) instrument_model = IonObject('InstrumentModel', **instrument_model_attrs) self.cabled_instrument_model_id = self.ims.create_instrument_model( instrument_model) # Create the cabled sites and link them appropriately. platform_site = IonObject('', **platform_site_attrs) self.cabled_platform_site_id = self.oms.create_platform_site(platform_site) self.oms.assign_site_to_site(self.cabled_platform_site_id, self.obs_id) instrument_site = IonObject('', **instrument_site_attrs) self.cabled_instrument_site_id = self.oms.create_instrument_site(instrument_site) self.oms.assign_site_to_site(self.cabled_instrument_site_id, self.cabled_platform_site_id) # Assign models to available sites. self.oms.assign_platform_model_to_platform_site( self.cabled_platform_model_id, self.cabled_platform_site_id) self.oms.assign_instrument_model_to_instrument_site( self.cabled_instrument_model_id, self.cabled_instrument_site_id) def _create_uncabled_resources(self): """ Create preexisting infrastructure for the uncabled test environment: sites, deployments, models. """ pass def test_cabled_device_activation(self): """ Test registration and activation of cabled device infrastructure. """ pass def test_uncabled_device_activation(self): """ Test registration and activation of uncabled device infrastructure. """ pass
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) def test_get_data_product_provenance_report(self): #Create a test device device_obj = Device(name='Device1', description='test instrument site') device_id, _ = self.rrclient.create(device_obj) self.addCleanup(self.rrclient.delete, device_id) #Create a test DataProduct data_product1_obj = DataProduct(name='DataProduct1', description='test data product 1') data_product1_id, _ = self.rrclient.create(data_product1_obj) self.addCleanup(self.rrclient.delete, data_product1_id) #Create a test DataProcess data_process_obj = DataProcess(name='DataProcess', description='test data process') data_process_id, _ = self.rrclient.create(data_process_obj) self.addCleanup(self.rrclient.delete, data_process_id) #Create a second test DataProduct data_product2_obj = DataProduct(name='DataProduct2', description='test data product 2') data_product2_id, _ = self.rrclient.create(data_product2_obj) self.addCleanup(self.rrclient.delete, data_product2_id) #Create a test DataProducer data_producer_obj = DataProducer(name='DataProducer', description='test data producer') data_producer_id, rev = self.rrclient.create(data_producer_obj) #Link the DataProcess to the second DataProduct manually assoc_id, _ = self.rrclient.create_association(subject=data_process_id, predicate=PRED.hasInputProduct, object=data_product2_id) self.addCleanup(self.rrclient.delete_association, assoc_id) # Register the instrument and process. This links the device and the data process # with their own producers self.damsclient.register_instrument(device_id) self.addCleanup(self.damsclient.unregister_instrument, device_id) self.damsclient.register_process(data_process_id) self.addCleanup(self.damsclient.unregister_process, data_process_id) #Manually link the first DataProduct with the test DataProducer assoc_id, _ = self.rrclient.create_association(subject=data_product1_id, predicate=PRED.hasDataProducer, object=data_producer_id) #Get the DataProducer linked to the DataProcess (created in register_process above) #Associate that with with DataProduct1's DataProducer data_process_producer_ids, _ = self.rrclient.find_objects(subject=data_process_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True) assoc_id, _ = self.rrclient.create_association(subject=data_process_producer_ids[0], predicate=PRED.hasParent, object=data_producer_id) self.addCleanup(self.rrclient.delete_association, assoc_id) #Get the DataProducer linked to the Device (created in register_instrument #Associate that with the DataProcess's DataProducer device_producer_ids, _ = self.rrclient.find_objects(subject=device_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True) assoc_id, _ = self.rrclient.create_association(subject=data_producer_id, predicate=PRED.hasParent, object=device_producer_ids[0]) #Create the links between the Device, DataProducts, DataProcess, and all DataProducers self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=data_product1_id) self.addCleanup(self.damsclient.unassign_data_product, device_id, data_product1_id) self.damsclient.assign_data_product(input_resource_id=data_process_id, data_product_id=data_product2_id) self.addCleanup(self.damsclient.unassign_data_product, data_process_id, data_product2_id) #Traverse through the relationships to get the links between objects res = self.dpmsclient.get_data_product_provenance_report(data_product2_id) #Make sure there are four keys self.assertEqual(len(res.keys()), 4) parent_count = 0 config_count = 0 for v in res.itervalues(): if 'parent' in v: parent_count += 1 if 'config' in v: config_count += 1 #Make sure there are three parents and four configs self.assertEqual(parent_count, 3) self.assertEqual(config_count, 4) @unittest.skip('This test is obsolete with new framework') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict' ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [parsed_config] ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: input_data_products = [ctd_parsed_data_product] output_data_products = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L0_all_dprocdef_id, in_data_product_ids = input_data_products, out_data_product_ids = output_data_products ) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_conductivity_dprocdef_id, in_data_product_ids = [ctd_l0_conductivity_output_dp_id], out_data_product_ids = [ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_pressure_dprocdef_id, in_data_product_ids = [ctd_l0_pressure_output_dp_id], out_data_product_ids = [ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_temperature_dprocdef_id, in_data_product_ids = [ctd_l0_temperature_output_dp_id], out_data_product_ids = [ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L2_salinity_dprocdef_id, in_data_product_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], out_data_product_ids = [ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: in_dp_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id] out_dp_ids = [ctd_l2_density_output_dp_id] l2_density_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L2_density_dprocdef_id, in_data_product_ids = in_dp_ids, out_data_product_ids = out_dp_ids) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) self.assertEqual(1, len(extended_product.data_processes) ) self.assertEqual(3, len(extended_product.process_input_data_products) ) # log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) # log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id) print results #------------------------------- # Cleanup #------------------------------- self.dpmsclient.delete_data_product(ctd_parsed_data_product) self.dpmsclient.delete_data_product(log_data_product_id) self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() #self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources( RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def cleanupprocs(self): stm = os.popen('ps -e | grep ion.agents.port.logger_process') procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) stm = os.popen('ps -e | grep ion.agents.instrument.zmq_driver_process') procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) # stm = os.popen('rm /tmp/*.pid.txt') @unittest.skip( "Deprecated by IngestionManagement refactor, timeout on start inst agent?" ) def test_deploy_activate_full(self): # ensure no processes or pids are left around by agents or Sims #self.cleanupprocs() self.loggerpids = [] #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD) try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) #------------------------------- # Create Instrument Site #------------------------------- instrumentSite_obj = IonObject(RT.InstrumentSite, name='instrumentSite1', description="SBE37IMInstrumentSite") try: instrumentSite_id = self.omsclient.create_instrument_site( instrument_site=instrumentSite_obj, parent_id='') except BadRequest as ex: self.fail("failed to create new InstrumentSite: %s" % ex) print 'test_deployAsPrimaryDevice: new instrumentSite id = ', instrumentSite_id self.omsclient.assign_instrument_model_to_instrument_site( instModel_id, instrumentSite_id) #------------------------------- # Logical Transform: Output Data Products #------------------------------- # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Old InstrumentDevice #------------------------------- instDevice_obj = IonObject( RT.InstrumentDevice, name='SBE37IMDeviceYear1', description="SBE37IMDevice for the FIRST year of deployment", serial_number="12345") try: oldInstDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, oldInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print 'test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ', oldInstDevice_id self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='SiteDataProduct', description='SiteDataProduct', temporal_domain=tdom, spatial_domain=sdom) instrument_site_output_dp_id = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=instrument_site_output_dp_id) #self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects( instrument_site_output_dp_id, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects( instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='first deployment') oldDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id) #------------------------------- # Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information # cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear1', description="SBE37IMAgentInstanceYear1", port_agent_config=port_agent_config, stream_configurations=[raw_config, parsed_config]) oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, oldInstDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() #------------------------------- # Create CTD Parsed as the Year 1 data product and attach to instrument #------------------------------- print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year1', description='ctd stream test year 1', temporal_domain=tdom, spatial_domain=sdom) ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year1 self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects( ctd_parsed_data_product_year1, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams1 = ', stream_ids #------------------------------- # Create New InstrumentDevice #------------------------------- instDevice_obj_2 = IonObject( RT.InstrumentDevice, name='SBE37IMDeviceYear2', description="SBE37IMDevice for the SECOND year of deployment", serial_number="67890") try: newInstDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj_2) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, newInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print 'test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ', newInstDevice_id #set the LCSTATE self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE) instDevice_obj_2 = self.rrclient.read(newInstDevice_id) log.debug( "test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate)) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='second deployment') newDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id) #------------------------------- # Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4004, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 4005, 'data_port': 4006, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear2', description="SBE37IMAgentInstanceYear2", port_agent_config=port_agent_config) newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, newInstDevice_id) #------------------------------- # Create CTD Parsed as the Year 2 data product #------------------------------- dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year2', description='ctd stream test year 2', temporal_domain=tdom, spatial_domain=sdom) ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year2 self.damsclient.assign_data_product( input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects( ctd_parsed_data_product_year2, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug( "test_deployAsPrimaryDevice: create data process definition ctd_L0_all" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new ctd_L0_all data process definition: %s" % ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name='L0_Conductivity', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity') outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name='L0_Pressure', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure') outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name='L0_Temperature', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature') self.out_prod_dict = {} log.debug( "test_deployAsPrimaryDevice: create output data product L0 conductivity" ) ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_conductivity_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.out_prod_dict['conductivity'] = ctd_l0_conductivity_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug( "test_deployAsPrimaryDevice: create output data product L0 pressure" ) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_pressure_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.out_prod_dict['pressure'] = ctd_l0_pressure_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug( "test_deployAsPrimaryDevice: create output data product L0 temperature" ) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_temperature_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.out_prod_dict['temperature'] = ctd_l0_temperature_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product) #------------------------------- log.debug( "test_deployAsPrimaryDevice: create L0 all data_process start") try: out_data_products = self.out_prod_dict.values() ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], out_data_products) self.dataprocessclient.activate_data_process( ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "test_deployAsPrimaryDevice: create L0 all data_process return") #-------------------------------- # Activate the deployment #-------------------------------- self.omsclient.activate_deployment(oldDeployment_id) #------------------------------- # Launch InstrumentAgentInstance Sim1, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=oldInstAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=oldInstAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance( oldInstAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, oldInstDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance( oldInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent1_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim1 = ResourceAgentClient('iaclient Sim1', name=gate.process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim1 %s', self._ia_client_sim1 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1)) #------------------------------- # Launch InstrumentAgentInstance Sim2, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=newInstAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=newInstAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance( newInstAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, oldInstDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance( newInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent2_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim2 = ResourceAgentClient('iaclient Sim2', name=gate.process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim2 %s', self._ia_client_sim2 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2)) #------------------------------- # Streaming Sim1 (old instrument) #------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active %s", str(reply)) self.assertTrue(reply) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim1.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from START_AUTOSAMPLE: %s", str(retval)) #------------------------------- # Streaming Sim 2 (new instrument) #------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim2.execute_agent(cmd) log.debug( "test_deployAsPrimaryDevice: return value from go_active_sim2 %s", str(reply)) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim2.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s", str(retval)) gevent.sleep(10) #------------------------------- # Shutdown Sim1 (old instrument) #------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(5) #------------------------------- # Shutdown Sim2 (old instrument) #------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset_sim2 ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset_sim2 %s", str(reply)) time.sleep(5)