class TestActivateRSNVel3DInstrument(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateRSNVel3DInstrument, self).setUp() config = DotDict() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid @attr('LOCOINT') @unittest.skip('under construction') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 180}}}) def test_activate_rsn_vel3d(self): log.info( "--------------------------------------------------------------------------------------------------------" ) # load_parameter_scenarios self.container.spawn_process( "Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories= "ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='Vel3DMModel', description="Vel3DMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('test_activate_rsn_vel3d new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') vel3d_b_sample = StreamConfiguration( stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample') vel3d_b_engineering = StreamConfiguration( stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering') RSN_VEL3D_01 = { 'DEV_ADDR': "10.180.80.6", 'DEV_PORT': 2101, 'DATA_PORT': 1026, 'CMD_PORT': 1025, 'PA_BINARY': "port_agent" } # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='Vel3DAgent', description="Vel3DAgent", driver_uri= "http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg", stream_configurations=[ raw_config, vel3d_b_sample, vel3d_b_engineering ]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='Vel3DDevice', description="Vel3DDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s ", instDevice_id) port_agent_config = { 'device_addr': '10.180.80.6', 'device_port': 2101, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 1025, 'data_port': 1026, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='Vel3DAgentInstance', description="Vel3DAgentInstance", port_agent_config=port_agent_config, alerts=[]) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_sample', id_only=True) parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition( name='vel3d_b_sample', parameter_dictionary_id=parsed_sample_pdict_id) parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_engineering', id_only=True) parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition( name='vel3d_b_engineering', parameter_dictionary_id=parsed_eng_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='vel3d_b_sample', description='vel3d_b_sample') sample_data_product_id = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_sample_stream_def_id) log.debug('new dp_id = %s', sample_data_product_id) self.dpclient.activate_data_product_persistence( data_product_id=sample_data_product_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=sample_data_product_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasStream, None, True) log.debug('sample_data_product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for sample_data_product = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('vel3d_b_sample', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='vel3d_b_engineering', description='vel3d_b_engineering') eng_data_product_id = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id) log.debug('new dp_id = %s', eng_data_product_id) self.dpclient.activate_data_product_persistence( data_product_id=eng_data_product_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=eng_data_product_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('test_activate_rsn_vel3d Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) def check_state(label, desired_state): actual_state = self._ia_client.get_agent_state() log.debug("%s instrument agent is in state '%s'", label, actual_state) self.assertEqual(desired_state, actual_state) log.debug("test_activate_rsn_vel3d: got ia client %s", str(self._ia_client)) check_state("just-spawned", ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: initialize %s", str(retval)) check_state("initialized", ResourceAgentState.INACTIVE) log.debug("test_activate_rsn_vel3d Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return value from go_active %s", str(reply)) check_state("activated", ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending go_active command %s", str(state)) # cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: run %s", str(reply)) check_state("commanded", ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending run command %s", str(state)) # cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # gevent.sleep(5) # # cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state)) # # cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.STOPPED, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RESUME) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.IDLE, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RUN) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) log.debug("test_activate_rsn_vel3d: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return from reset %s", str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print())) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(sample_data_product_id) self.dpclient.delete_data_product(eng_data_product_id) self.dpclient.delete_data_product(data_product_id2)
class TestInstrumentAlerts(IonIntegrationTestCase): pdict_id = None def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.catch_alert = gevent.queue.Queue() def _create_instrument_model(self): instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) self.addCleanup(self.imsclient.delete_instrument_model, instModel_id) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.addCleanup(self.imsclient.delete_instrument_agent, instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) self.addCleanup(self.imsclient.delete_instrument_device, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_stream_alarms(self, instDevice_id): #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ temp_alert_def1 = { 'name': 'temperature_warning_interval temp below 25', 'stream_name': 'parsed', 'description': 'temperature_warning_interval temp below 25', 'alert_type': StreamAlertType.WARNING, 'aggregate_type': AggregateStatusType.AGGREGATE_DATA, 'value_id': 'temp', 'resource_id': instDevice_id, 'origin_type': 'device', #'lower_bound' : 0, #'lower_rel_op' : '<', # temp 'upper_rel_op': '<=', 'upper_bound': 25, 'alert_class': 'IntervalAlert' } temp_alert_def2 = { 'name': 'temperature_warning_interval temp below 50', 'stream_name': 'parsed', 'description': 'temperature_warning_interval temp below 50', 'alert_type': StreamAlertType.WARNING, 'aggregate_type': AggregateStatusType.AGGREGATE_DATA, 'value_id': 'temp', 'resource_id': instDevice_id, 'origin_type': 'device', #'lower_bound' : 25, #'lower_rel_op' : '<', # temp 'upper_rel_op': '<=', 'upper_bound': 50, 'alert_class': 'IntervalAlert' } temp_alert_def3 = { 'name': 'temperature_warning_interval temp below 75', 'stream_name': 'parsed', 'description': 'temperature_warning_interval temp below 75', 'alert_type': StreamAlertType.WARNING, 'aggregate_type': AggregateStatusType.AGGREGATE_DATA, 'value_id': 'temp', 'resource_id': instDevice_id, 'origin_type': 'device', #'lower_bound' : 50, #'lower_rel_op' : '<', # temp 'upper_rel_op': '<=', 'upper_bound': 75, 'alert_class': 'IntervalAlert' } late_data_alert_def = { 'name': 'late_data_warning', 'stream_name': 'parsed', 'description': 'Expected data has not arrived.', 'alert_type': StreamAlertType.WARNING, 'aggregate_type': AggregateStatusType.AGGREGATE_COMMS, 'value_id': None, 'resource_id': instDevice_id, 'origin_type': 'device', 'time_delta': 2, 'alert_class': 'LateDataAlert' } return [ temp_alert_def1, temp_alert_def2, temp_alert_def3, late_data_alert_def ] def _create_instrument_agent_instance(self, instAgent_id, instDevice_id): port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } self.all_alerts = self._create_instrument_stream_alarms(instDevice_id) instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config, alerts=self.all_alerts) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) self.addCleanup(self.imsclient.delete_instrument_agent_instance, instAgentInstance_id) return instAgentInstance_id def test_alerts(self): # # test that with the 4009 sim we can get a late data alert # as well as alerts for out of range for > 25, > 50, and > 75 # as well as the ALL_CLEAR alerts for each of them. # #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) # It is necessary for the instrument device to be associated with atleast one output data product parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) # We are creating two data products here, one for parsed and another raw dp_obj_parsed = IonObject( RT.DataProduct, name='parsed_data_product', description='Parsed output data product for instrument') dp_obj_raw = IonObject( RT.DataProduct, name='raw_data_prod', description='Raw output data product for instrument') parsed_out_data_prod_id = self.dataproductclient.create_data_product( data_product=dp_obj_parsed, stream_definition_id=parsed_stream_def_id) raw_out_data_prod_id = self.dataproductclient.create_data_product( data_product=dp_obj_raw, stream_definition_id=raw_stream_def_id) self.addCleanup(self.dataproductclient.delete_data_product, parsed_out_data_prod_id) self.addCleanup(self.dataproductclient.delete_data_product, raw_out_data_prod_id) self.dataproductclient.activate_data_product_persistence( data_product_id=parsed_out_data_prod_id) self.dataproductclient.activate_data_product_persistence( data_product_id=raw_out_data_prod_id) # todo: note that the generated config on the instruments will be done for both raw and parsed stream defs since these two data products constructed with each are associated as output data products with the instrument # todo: if the config is not generated for a stream def, then the instrument agent will complain if the simulator generates data corresponding to a stream def that is not there in the stream config as a mentioned stream def self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=parsed_out_data_prod_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=raw_out_data_prod_id) log.debug("assigned instdevice id: %s to data product: %s", instDevice_id, raw_out_data_prod_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance( instAgent_id, instDevice_id) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) # Wait for instrument agent to spawn gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Set up the subscriber to catch the alert event #------------------------------------------------------------------------------------- def callback_for_alert(event, *args, **kwargs): log.debug("caught an alert: %s", event) self.catch_alert.put(event) self.event_subscriber = EventSubscriber( event_type='DeviceStatusAlertEvent', origin=instDevice_id, callback=callback_for_alert) self.event_subscriber.start() self.addCleanup(self.event_subscriber.stop) #------------------------------------------------------------------------------------- # Running the instrument.... #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) # Prevent this test from hanging indefinitely until # OOIION-1313 is resolved timeout_val = 90 with gevent.Timeout( timeout_val, Exception('Agent failed to initialize after %fs' % timeout_val)): reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) with gevent.Timeout( timeout_val, Exception('Agent failed to go active after %fs' % timeout_val)): reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) with gevent.Timeout( timeout_val, Exception('Agent failed to get resource after %fs' % timeout_val)): retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) with gevent.Timeout( timeout_val, Exception('Agent failed to run after %fs' % timeout_val)): reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) with gevent.Timeout( timeout_val, Exception('Agent failed to start autosample after %fs' % timeout_val)): retval = self._ia_client.execute_resource(cmd) got_bad_temp = [False, False, False, False] got_late_data = False got_temp_all_clear = [False, False, False, False] runtime = 0 starttime = time.time() caught_events = [] while (got_bad_temp[0] == False or got_bad_temp[1] == False or got_bad_temp[2] == False or got_temp_all_clear[0] == False or got_temp_all_clear[1] == False or got_temp_all_clear[2] == False or got_late_data == False) and runtime < 120: a = self.catch_alert.get(timeout=180) caught_events.append(a) if a.name == 'temperature_warning_interval temp below 25': if a.sub_type == 'WARNING' and \ a.values[0] > 25: got_bad_temp[0] = True log.error(str(a.values[0]) + " should be above 25") elif a.sub_type == 'ALL_CLEAR': got_temp_all_clear[0] = True log.debug("25 ALL_CLEAR") if a.name == 'temperature_warning_interval temp below 50': if a.sub_type == 'WARNING' and \ a.values[0] > 50: got_bad_temp[1] = True log.error(str(a.values[0]) + " should be above 50") elif a.sub_type == 'ALL_CLEAR': got_temp_all_clear[1] = True log.debug("50 ALL_CLEAR") if a.name == 'temperature_warning_interval temp below 75': if a.sub_type == 'WARNING' and \ a.values[0] > 75: got_bad_temp[2] = True log.error(str(a.values[0]) + " should be above 75") elif a.sub_type == 'ALL_CLEAR': got_temp_all_clear[2] = True log.debug("75 ALL_CLEAR") if a.name == 'late_data_warning' and \ a.description == 'Expected data has not arrived.': got_late_data = True log.debug("late value") runtime = time.time() - starttime log.debug("caught_events: %s", [c.name for c in caught_events]) for c in caught_events: self.assertIn(c.name, [ 'temperature_warning_interval temp below 25', 'temperature_warning_interval temp below 50', 'temperature_warning_interval temp below 75', 'late_data_warning' ]) self.assertEqual(c.origin, instDevice_id) self.assertEqual(c.type_, 'DeviceStatusAlertEvent') self.assertEqual(c.origin_type, 'InstrumentDevice') self.assertTrue(got_bad_temp) #simply log the state of the late data flag and the simulator does not seem to be consistent. log.debug("test_alerts late data alert flag: %s", got_late_data)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node) def test_createDataProcess(self): #--------------------------------------------------------------------------- # Data Process Definition #--------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # Make assertion on the newly registered data process definition data_process_definition = self.rrclient.read(dprocdef_id) self.assertEquals(data_process_definition.name, 'ctd_L0_all') self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams') self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all') self.assertEquals(data_process_definition.class_name, 'ctd_L0_all') # Read the data process definition using data process management and make assertions dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id) self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all') self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all') #--------------------------------------------------------------------------- # Create an input instrument #--------------------------------------------------------------------------- instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) # Register the instrument so that the data producer and stream object are created data_producer_id = self.damsclient.register_instrument(instrument_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id ) # Assert that the link between the stream definition and the data process definition was done assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True) self.assertIsNotNone(assocs) #--------------------------------------------------------------------------- # Input Data Product #--------------------------------------------------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() input_dp_obj = IonObject( RT.DataProduct, name='InputDataProduct', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test') #Make assertions on the input data product created input_dp_obj = self.rrclient.read(input_dp_id) self.assertEquals(input_dp_obj.name, 'InputDataProduct') self.assertEquals(input_dp_obj.description, 'some new dp') self.damsclient.assign_data_product(instrument_id, input_dp_id) # Retrieve the stream via the DataProduct->Stream associations stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True) self.in_stream_id = stream_ids[0] #--------------------------------------------------------------------------- # Output Data Product #--------------------------------------------------------------------------- outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' ) outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' ) outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' ) self.output_products={} output_dp_obj = IonObject(RT.DataProduct, name='conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id) self.output_products['conductivity'] = output_dp_id_1 output_dp_obj = IonObject(RT.DataProduct, name='pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id) self.output_products['pressure'] = output_dp_id_2 output_dp_obj = IonObject(RT.DataProduct, name='temperature', description='transform output ', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id) self.output_products['temperature'] = output_dp_id_3 #--------------------------------------------------------------------------- # Create the data process #--------------------------------------------------------------------------- def _create_data_process(): dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products) return dproc_id dproc_id = _create_data_process() # Make assertions on the data process created data_process = self.dataprocessclient.read_data_process(dproc_id) # Assert that the data process has a process id attached self.assertIsNotNone(data_process.process_id) # Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute self.assertIsNotNone(data_process.input_subscription_id) output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True) self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3])) @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}}) def test_createDataProcessUsingSim(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg") instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) #------------------------------- # Create CTD Raw as the second data product #------------------------------- raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj.name = 'ctd_raw' ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # Create listener for data process events and verify that events are received. #------------------------------- # todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events #todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_id = data_process.process_id self.addCleanup(self.process_dispatcher.cancel_process, process_id) #------------------------------- # Wait until the process launched in the create_data_process() method is actually running, before proceeding further in this test #------------------------------- gate = ProcessStateGate(self.process_dispatcher.read_process, process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The data process (%s) did not spawn in 30 seconds" % process_id) #------------------------------- # Retrieve a list of all data process defintions in RR and validate that the DPD is listed #------------------------------- # todo: Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions # todo: data topics are being handled by pub sub at the level of streams self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) #todo: check that activate event is received L4-CI-SA-RQ-367 #todo... (it looks like no event is being published when the data process is activated... so below, we just check for now # todo... that the subscription is indeed activated) (Swarbhanu) # todo: monitor process to see if it is active (sa-rq-182) ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id) input_subscription_id = ctd_l0_all_data_process.input_subscription_id subs = self.rrclient.read(input_subscription_id) self.assertTrue(subs.activated) # todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the extended resources for data process definition and for data process #------------------------------- extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) self.assertEqual(1, len(extended_process_definition.data_processes)) log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition)) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEqual(1, len(extended_process.input_data_products)) log.debug("test_createDataProcess: extended_process %s", str(extended_process)) ################################ Test the removal of data processes ################################## #------------------------------------------------------------------- # Test the deleting of the data process #------------------------------------------------------------------- # Before deleting, get the input streams, output streams and the subscriptions so that they can be checked after deleting # dp_obj_1 = self.rrclient.read(ctd_l0_all_data_process_id) # input_subscription_id = dp_obj_1.input_subscription_id # out_prods, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) # in_prods, _ = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasInputProduct, id_only=True) # in_streams = [] # for in_prod in in_prods: # streams, _ = self.rrclient.find_objects(in_prod, PRED.hasStream, id_only=True) # in_streams.extend(streams) # out_streams = [] # for out_prod in out_prods: # streams, _ = self.rrclient.find_objects(out_prod, PRED.hasStream, id_only=True) # out_streams.extend(streams) # Deleting the data process self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id) # Check that the data process got removed. Check the lcs state. It should be retired dp_obj = self.rrclient.read(ctd_l0_all_data_process_id) self.assertEquals(dp_obj.lcstate, LCS.RETIRED) # Check for process defs still attached to the data process dpd_assn_ids = self.rrclient.find_associations(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcessDefinition, id_only=True) self.assertEquals(len(dpd_assn_ids), 0) # Check for output data product still attached to the data process out_products, assocs = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) self.assertEquals(len(out_products), 0) self.assertEquals(len(assocs), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check of the data process has been deactivated self.assertIsNone(dp_obj.input_subscription_id) # Read the original subscription id of the data process and check that it has been deactivated with self.assertRaises(NotFound): self.pubsubclient.read_subscription(input_subscription_id) #------------------------------------------------------------------- # Delete the data process definition #------------------------------------------------------------------- # before deleting, get the process definition being associated to in order to be able to check later if the latter gets deleted as it should proc_def_ids, proc_def_asocs = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasProcessDefinition) self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id) # check that the data process definition has been retired dp_proc_def = self.rrclient.read(ctd_L0_all_dprocdef_id) self.assertEquals(dp_proc_def.lcstate, LCS.RETIRED) # Check for old associations of this data process definition proc_defs, proc_def_asocs = self.rrclient.find_objects(ctd_L0_all_dprocdef_id, PRED.hasProcessDefinition) self.assertEquals(len(proc_defs), 0) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject= ctd_L0_all_dprocdef_id, id_only=True) self.assertEquals(len(obj_assns), 0) ################################ Test the removal of data processes ################################## # Try force delete... This should simply delete the associations and the data process object # from the resource registry #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id) #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process definition #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dmpsclient = DataProductManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.psmsclient = PubsubManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) # create missing data process definition self.dsmsclient = DataProcessManagementServiceClient( node=self.container.node) dpd_obj = IonObject( RT.DataProcessDefinition, name=LOGICAL_TRANSFORM_DEFINITION_NAME, description="normally in preload", module='ion.processes.data.transforms.logical_transform', class_name='logical_transform') self.dsmsclient.create_data_process_definition(dpd_obj) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources( RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013, 1, 1)) end = IonTime(datetime.datetime(2014, 1, 1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj)) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.RR2.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self): #------------------------------------------------------------------------------------- # Create platform site, platform device, platform model #------------------------------------------------------------------------------------- platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') platform_site_id = self.omsclient.create_platform_site( platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device( platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model( platform_model__obj) #------------------------------------------------------------------------------------- # Create instrument site #------------------------------------------------------------------------------------- instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site( instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='Log Data Product', description='some new dp', temporal_domain=tdom, spatial_domain=sdom) out_log_data_product_id = self.dmpsclient.create_data_product( dp_obj, ctd_stream_def_id) #---------------------------------------------------------------------------------------------------- # Start the transform (a logical transform) that acts as an instrument site #---------------------------------------------------------------------------------------------------- self.omsclient.create_site_data_product( site_id=instrument_site_id, data_product_id=out_log_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument device #---------------------------------------------------------------------------------------------------- instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) dp_obj = IonObject(RT.DataProduct, name='Instrument Data Product', description='some new dp', temporal_domain=tdom, spatial_domain=sdom) inst_data_product_id = self.dmpsclient.create_data_product( dp_obj, ctd_stream_def_id) #assign data products appropriately self.damsclient.assign_data_product( input_resource_id=instrument_device_id, data_product_id=inst_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument model #---------------------------------------------------------------------------------------------------- instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model( instrument_model_obj) #---------------------------------------------------------------------------------------------------- # Create a deployment object #---------------------------------------------------------------------------------------------------- start = IonTime(datetime.datetime(2013, 1, 1)) end = IonTime(datetime.datetime(2014, 1, 1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_platform_model_to_platform_device( res.platform_model_id, res.platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_platform_model_to_platform_site( res.platform_model_id, res.platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("adding instrument site and device to deployment") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("adding platform site and device to deployment") self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id) self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug( "activating deployment without site+device models, expecting fail") self.assert_deploy_fail( res.deployment_id, "Expected at least 1 model for InstrumentSite") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug( "activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, "Expected 1 model for InstrumentDevice") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug( "activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, "No sites were found in the deployment") #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) log.debug( "activating deployment without device models, expecting fail") self.assert_deploy_fail( res.deployment_id, "The set of devices could not be mapped to the set of sites") def assert_deploy_fail(self, deployment_id, fail_message="did not specify fail_message"): with self.assertRaises(BadRequest) as cm: self.omsclient.activate_deployment(deployment_id) self.assertIn(fail_message, cm.exception.message)
class VisualizationIntegrationTestHelper(IonIntegrationTestCase): def create_ctd_input_stream_and_data_product(self, data_product_name='ctd_parsed'): cc = self.container assertions = self.assertTrue # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.workflowclient = WorkflowManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node) self.vis_client = VisualizationServiceClient(node=self.container.node) #------------------------------- # Create CTD Parsed as the initial data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data') log.debug('Creating new CDM data product with a stream definition') craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name=data_product_name, description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product_id = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug('new ctd_parsed_data_product_id = %s' % ctd_parsed_data_product_id) #Only ever need one device for testing purposes. instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='SBE37IMDevice') if instDevice_obj: instDevice_id = instDevice_obj[0]._id else: instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product_id) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0 ) ctd_stream_id = stream_ids[0] return ctd_stream_id, ctd_parsed_data_product_id def create_data_product(self, dp_name = "", dp_description = ""): craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() # this creates a ParameterDictionary object parameter_dictionary = parameter_dictionary.dump() # this returns a python dictionary data_prod_obj = IonObject(RT.DataProduct, name=dp_name, description=dp_description, temporal_domain = tdom, spatial_domain = sdom) data_prod_id = self.create_data_product(data_prod_obj, stream_definition_id, parameter_dictionary) return data_prod_id, data_prod_obj def start_simple_input_stream_process(self, ctd_stream_id): return self.start_input_stream_process(ctd_stream_id) def start_sinusoidal_input_stream_process(self, ctd_stream_id): return self.start_input_stream_process(ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher', 'SinusoidalCtdPublisher') def start_input_stream_process(self, ctd_stream_id, module = 'ion.processes.data.ctd_stream_publisher', class_name= 'SimpleCtdPublisher'): ### ### Start the process for producing the CTD data ### # process definition for the ctd simulator... producer_definition = ProcessDefinition() producer_definition.executable = { 'module':module, 'class':class_name } ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(process_definition=producer_definition) # Start the ctd simulator to produce some data configuration = { 'process':{ 'stream_id':ctd_stream_id, } } ctd_sim_pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration) return ctd_sim_pid def start_output_stream_and_listen(self, ctd_stream_id, data_product_stream_ids, message_count_per_stream=10): cc = self.container assertions = self.assertTrue ### ### Make a subscriber in the test to listen for transformed data ### salinity_subscription_id = self.pubsubclient.create_subscription( query=StreamQuery(data_product_stream_ids), exchange_name = 'workflow_test', exchange_point = 'science_data', name = "test workflow transformations", ) pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, container=cc) result = gevent.event.AsyncResult() results = [] message_count = len(data_product_stream_ids) * message_count_per_stream def message_received(message, headers): # Heads results.append(message) if len(results) >= message_count: #Only wait for so many messages - per stream result.set(True) subscriber = subscriber_registrar.create_subscriber(exchange_name='workflow_test', callback=message_received) subscriber.start() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id) #Start the input stream process if ctd_stream_id is not None: ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) # Assert that we have received data assertions(result.get(timeout=30)) # stop the flow parse the messages... if ctd_stream_id is not None: self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id) subscriber.stop() return results def validate_messages(self, results): cc = self.container assertions = self.assertTrue first_salinity_values = None for message in results: rdt = RecordDictionaryTool.load_from_granule(message) try: temp = get_safe(rdt, 'temp') # psd = PointSupplementStreamParser(stream_definition=self.ctd_stream_def, stream_granule=message) # temp = psd.get_values('temperature') # log.info(psd.list_field_names()) except KeyError as ke: temp = None if temp is not None: assertions(isinstance(temp, numpy.ndarray)) log.info( 'temperature=' + str(numpy.nanmin(temp))) first_salinity_values = None else: #psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message) #log.info( psd.list_field_names()) # Test the handy info method for the names of fields in the stream def #assertions('salinity' in psd.list_field_names()) # you have to know the name of the coverage in stream def salinity = get_safe(rdt, 'salinity') #salinity = psd.get_values('salinity') log.info( 'salinity=' + str(numpy.nanmin(salinity))) # Check to see if salinity has values assertions(salinity != None) assertions(isinstance(salinity, numpy.ndarray)) assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0 if first_salinity_values is None: first_salinity_values = salinity.tolist() else: second_salinity_values = salinity.tolist() assertions(len(first_salinity_values) == len(second_salinity_values)) for idx in range(0,len(first_salinity_values)): assertions(first_salinity_values[idx]*2.0 == second_salinity_values[idx]) def validate_data_ingest_retrieve(self, dataset_id): assertions = self.assertTrue self.data_retriever = DataRetrieverServiceClient(node=self.container.node) #validate that data was ingested replay_granule = self.data_retriever.retrieve_last_granule(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) salinity = get_safe(rdt, 'salinity') assertions(salinity != None) #retrieve all the granules from the database and check the values replay_granule_all = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule_all) for k, v in rdt.iteritems(): if k == 'salinity': for val in numpy.nditer(v): assertions(val > 0) def create_salinity_data_process_definition(self): # Salinity: Data Process Definition #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='ctd_salinity') if len(dpd) > 0: return dpd[0] log.debug("Create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_salinity', description='create a salinity data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Excpetion as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) # create a stream definition for the data from the salinity Transform sal_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='Salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(sal_stream_def_id, ctd_L2_salinity_dprocdef_id ) return ctd_L2_salinity_dprocdef_id def create_salinity_doubler_data_process_definition(self): #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler') if len(dpd) > 0: return dpd[0] # Salinity Doubler: Data Process Definition log.debug("Create data process definition SalinityDoublerTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='salinity_doubler', description='create a salinity doubler data product', module='ion.processes.data.transforms.example_double_salinity', class_name='SalinityDoubler', process_source='SalinityDoubler source code here...') try: salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Exception as ex: self.fail("failed to create new SalinityDoubler data process definition: %s" %ex) # create a stream definition for the data from the salinity Transform salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler') self.dataprocessclient.assign_stream_definition_to_data_process_definition(salinity_double_stream_def_id, salinity_doubler_dprocdef_id ) return salinity_doubler_dprocdef_id def create_transform_process(self, data_process_definition_id, data_process_input_dp_id): data_process_definition = self.rrclient.read(data_process_definition_id) # Find the link between the output Stream Definition resource and the Data Process Definition resource stream_ids,_ = self.rrclient.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True) if not stream_ids: raise Inconsistent("The data process definition %s is missing an association to an output stream definition" % data_process_definition._id ) process_output_stream_def_id = stream_ids[0] #Concatenate the name of the workflow and data process definition for the name of the data product output data_process_name = data_process_definition.name # Create the output data product of the transform transform_dp_obj = IonObject(RT.DataProduct, name=data_process_name,description=data_process_definition.description) transform_dp_id = self.dataproductclient.create_data_product(transform_dp_obj, process_output_stream_def_id) self.dataproductclient.activate_data_product_persistence(data_product_id=transform_dp_id) #last one out of the for loop is the output product id output_data_product_id = transform_dp_id # Create the transform data process log.debug("create data_process and start it") data_process_id = self.dataprocessclient.create_data_process(data_process_definition._id, [data_process_input_dp_id], {'output':transform_dp_id}) self.dataprocessclient.activate_data_process(data_process_id) #Find the id of the output data stream stream_ids, _ = self.rrclient.find_objects(transform_dp_id, PRED.hasStream, None, True) if not stream_ids: raise Inconsistent("The data process %s is missing an association to an output stream" % data_process_id ) return data_process_id, output_data_product_id def create_google_dt_data_process_definition(self): #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform') if len(dpd) > 0: return dpd[0] # Data Process Definition log.debug("Create data process definition GoogleDtTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='google_dt_transform', description='Convert data streams to Google DataTables', module='ion.processes.data.transforms.viz.google_dt', class_name='VizTransformGoogleDT', process_source='VizTransformGoogleDT source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Exception as ex: self.fail("failed to create new VizTransformGoogleDT data process definition: %s" %ex) # create a stream definition for the data from the stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformGoogleDT.outgoing_stream_def, name='VizTransformGoogleDT') self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id ) return procdef_id def validate_google_dt_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) gdt_data = get_safe(rdt, 'google_dt_components') # IF this granule does not contains google dt, skip if gdt_data == None: continue gdt = gdt_data[0] assertions(gdt['viz_product_type'] == 'google_dt' ) assertions(len(gdt['data_description']) >= 0) # Need to come up with a better check assertions(len(gdt['data_content']) >= 0) def create_mpl_graphs_data_process_definition(self): #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform') if len(dpd) > 0: return dpd[0] #Data Process Definition log.debug("Create data process definition MatplotlibGraphsTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='mpl_graphs_transform', description='Convert data streams to Matplotlib graphs', module='ion.processes.data.transforms.viz.matplotlib_graphs', class_name='VizTransformMatplotlibGraphs', process_source='VizTransformMatplotlibGraphs source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Exception as ex: self.fail("failed to create new VizTransformMatplotlibGraphs data process definition: %s" %ex) # create a stream definition for the data stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformMatplotlibGraphs.outgoing_stream_def, name='VizTransformMatplotlibGraphs') self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id ) return procdef_id def validate_mpl_graphs_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) graphs = get_safe(rdt, 'matplotlib_graphs') if graphs == None: continue for graph in graphs[0]: # At this point only dictionaries containing image data should be passed # For some reason non dictionary values are filtering through. if not isinstance(graph, dict): continue assertions(graph['viz_product_type'] == 'matplotlib_graphs' ) # check to see if the list (numpy array) contains actual images assertions(imghdr.what(graph['image_name'], h = graph['image_obj']) == 'png') def validate_vis_service_google_dt_results(self, results): assertions = self.assertTrue assertions(results) gdt_str = (results.lstrip("google.visualization.Query.setResponse(")).rstrip(")") assertions(len(gdt_str) > 0) return def validate_vis_service_mpl_graphs_results(self, results): assertions = self.assertTrue assertions(results) # check to see if the object passed is a dictionary with a valid image object in it image_format = results["content_type"].lstrip("image/") assertions(imghdr.what(results['image_name'], h = base64.decodestring(results['image_obj'])) == image_format) return
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.client = DataProductManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementService(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() @unittest.skip('not ready') def test_get_provenance(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_module='mi.instrument.seabird.sbe37smb.ooicore.driver', driver_class='SBE37Driver', comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) log.debug( 'new Stream Definition id = %s', instDevice_id) log.debug( 'Creating new CDM data product with a stream definition') craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) print 'TestDataProductProvenance: Data product streams1 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform', process_source='CTDL1ConductivityTransform source code here...') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform', process_source='CTDL1PressureTransform source code here...') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform', process_source='CTDL1TemperatureTransform source code here...') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform', process_source='DensityTransform source code here...') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products={} log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id, parameter_dictionary) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity = L1_conductivity_stream_definition() outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id ) outgoing_stream_l1_pressure = L1_pressure_stream_definition() outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id ) outgoing_stream_l1_temperature = L1_temperature_stream_definition() outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id, parameter_dictionary) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] ) self.loggerpids.append(pid) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id, parameter_dictionary) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0] ) self.loggerpids.append(pid) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id, parameter_dictionary) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0] ) self.loggerpids.append(pid) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id ) outgoing_stream_l2_density = L2_density_stream_definition() outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id, parameter_dictionary) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0] ) self.loggerpids.append(pid) log.debug("TestDataProductProvenance: create output data product L2 Density") ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id, parameter_dictionary) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0] ) self.loggerpids.append(pid) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProductProvenance: create L1 Conductivity data_process return") #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProductProvenance: create L1_Pressure data_process return") #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProductProvenance: create L1_Pressure data_process return") #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProductProvenance: create L2_salinity data_process return") #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProductProvenance: create L2_Density data_process return") provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
class TestPlatformInstrument(BaseIntTestPlatform): def setUp(self): self._start_container() self._pp = pprint.PrettyPrinter() log.debug("oms_uri = %s", OMS_URI) self.oms = CIOMSClientFactory.create_instance(OMS_URI) #url = OmsTestMixin.start_http_server() #log.debug("TestPlatformInstrument:setup http url %s", url) # #result = self.oms.event.register_event_listener(url) #log.debug("TestPlatformInstrument:setup register_event_listener result %s", result) self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.org_id = self.RR2.create(any_old(RT.Org)) log.debug("Org created: %s", self.org_id) # see _set_receive_timeout self._receive_timeout = 300 self.instrument_device_id = '' self.platform_device_id = '' self.platform_site_id = '' self.platform_agent_instance_id = '' self._pa_client = '' def done(): CIOMSClientFactory.destroy_instance(self.oms) event_notifications = OmsTestMixin.stop_http_server() log.info("event_notifications = %s" % str(event_notifications)) self.addCleanup(done) @unittest.skip('Must be run locally...') def test_platform_with_instrument_streaming(self): # # The following is with just a single platform and the single # instrument "SBE37_SIM_08", which corresponds to the one on port 4008. # #load the paramaters and the param dicts necesssary for the VEL3D log.debug( "load params------------------------------------------------------------------------------" ) self._load_params() log.debug( " _register_oms_listener------------------------------------------------------------------------------" ) self._register_oms_listener() #create the instrument device/agent/mode log.debug("---------- create_instrument_resources ----------") self._create_instrument_resources() #create the platform device, agent and instance log.debug("---------- create_platform_configuration ----------") self._create_platform_configuration('LPJBox_CI_Ben_Hall') self.rrclient.create_association(subject=self.platform_device_id, predicate=PRED.hasDevice, object=self.instrument_device_id) log.debug("---------- start_platform ----------") self._start_platform() self.addCleanup(self._stop_platform) # get everything in command mode: self._ping_agent() log.debug(" ---------- initialize ----------") self._initialize() _ia_client = ResourceAgentClient(self.instrument_device_id, process=FakeProcess()) state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) log.debug(" ---------- go_active ----------") self._go_active() state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) log.debug("---------- run ----------") self._run() gevent.sleep(2) log.debug(" ---------- _start_resource_monitoring ----------") self._start_resource_monitoring() gevent.sleep(2) # # # verify the instrument is command state: # state = ia_client.get_agent_state() # log.debug(" TestPlatformInstrument get_agent_state: %s", state) # self.assertEqual(state, ResourceAgentState.COMMAND) _stop_resource_monitoring log.debug(" ---------- _stop_resource_monitoring ----------") self._stop_resource_monitoring() gevent.sleep(2) log.debug(" ---------- go_inactive ----------") self._go_inactive() state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) self._reset() self._shutdown() def _get_platform_attributes(self): log.debug(" ----------get_platform_attributes ----------") attr_infos = self.oms.attr.get_platform_attributes( 'LPJBox_CI_Ben_Hall') log.debug('_get_platform_attributes: %s', self._pp.pformat(attr_infos)) attrs = attr_infos['LPJBox_CI_Ben_Hall'] for attrid, arrinfo in attrs.iteritems(): arrinfo['attr_id'] = attrid log.debug('_get_platform_attributes: %s', self._pp.pformat(attrs)) return attrs def _load_params(self): log.info(" ---------- load_params ----------") # load_parameter_scenarios self.container.spawn_process( "Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories= "ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) def _create_platform_configuration(self, platform_id, parent_platform_id=None): """ This method is an adaptation of test_agent_instance_config in test_instrument_management_service_integration.py @param platform_id @param parent_platform_id @return a DotDict with various of the constructed elements associated to the platform. """ param_dict_name = 'platform_eng_parsed' parsed_rpdict_id = self.dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) self.parsed_stream_def_id = self.pubsubclient.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_rpdict_id) driver_config = PLTFRM_DVR_CONFIG driver_config['attributes'] = self._get_platform_attributes( ) #self._platform_attributes[platform_id] #OMS returning an error for port.get_platform_ports #driver_config['ports'] = self._platform_ports[platform_id] log.debug("driver_config: %s", driver_config) # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': driver_config}) platform_agent_instance_obj.agent_config = { 'platform_config': { 'platform_id': 'LPJBox_CI_Ben_Hall', 'parent_platform_id': None } } self.platform_agent_instance_id = self.imsclient.create_platform_agent_instance( platform_agent_instance_obj) # agent creation platform_agent_obj = any_old( RT.PlatformAgent, { "stream_configurations": self._get_platform_stream_configs(), 'driver_module': PLTFRM_DVR_MOD, 'driver_class': PLTFRM_DVR_CLS }) platform_agent_id = self.imsclient.create_platform_agent( platform_agent_obj) # device creation self.platform_device_id = self.imsclient.create_platform_device( any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct) dp_id = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id) self.damsclient.assign_data_product( input_resource_id=self.platform_device_id, data_product_id=dp_id) self.dpclient.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.dpclient.delete_data_product, dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance( self.platform_agent_instance_id, self.platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition( platform_agent_id, self.platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource( self.platform_agent_instance_id, self.org_id) ####################################### # dataset log.debug('data product = %s', dp_id) stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) log.debug('Data product stream_ids = %s', stream_ids) stream_id = stream_ids[0] # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) ####################################### log.debug( '_create_platform_site_and_deployment platform_device_id: %s', self.platform_device_id) site_object = IonObject(RT.PlatformSite, name='PlatformSite1') self.platform_site_id = self.omsclient.create_platform_site( platform_site=site_object, parent_id='') log.debug('_create_platform_site_and_deployment site id: %s', self.platform_site_id) #create supporting objects for the Deployment resource # 1. temporal constraint # find current deployment using time constraints current_time = int(calendar.timegm(time.gmtime())) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) # 2. PlatformPort object which defines device to port map platform_port_obj = IonObject( OT.PlatformPort, reference_designator='GA01SUMO-FI003-01-CTDMO0999', port_type=PortTypeEnum.UPLINK, ip_address='0') # now create the Deployment deployment_obj = IonObject( RT.Deployment, name='TestPlatformDeployment', description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds], port_assignments={self.platform_device_id: platform_port_obj}) platform_deployment_id = self.omsclient.create_deployment( deployment=deployment_obj, site_id=self.platform_site_id, device_id=self.platform_device_id) log.debug('_create_platform_site_and_deployment deployment_id: %s', platform_deployment_id) deploy_obj2 = self.omsclient.read_deployment(platform_deployment_id) log.debug('_create_platform_site_and_deployment deploy_obj2 : %s', deploy_obj2) return self.platform_site_id, platform_deployment_id def _create_instrument_resources(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='VEL3D', description="VEL3D") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') vel3d_b_sample = StreamConfiguration( stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample') vel3d_b_engineering = StreamConfiguration( stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering') # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri= "http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg", stream_configurations=[ raw_config, vel3d_b_sample, vel3d_b_engineering ]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name='VEL3DDevice', description="VEL3DDevice", serial_number="12345") self.instrument_device_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, self.instrument_device_id) port_agent_config = { 'device_addr': '10.180.80.6', 'device_port': 2101, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 1025, 'data_port': 1026, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='VEL3DAgentInstance', description="VEL3DAgentInstance", port_agent_config=port_agent_config, alerts=[]) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, self.instrument_device_id) self._start_port_agent( self.imsclient.read_instrument_agent_instance( instAgentInstance_id)) vel3d_b_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_sample', id_only=True) vel3d_b_sample_stream_def_id = self.pubsubclient.create_stream_definition( name='vel3d_b_sample', parameter_dictionary_id=vel3d_b_sample_pdict_id) vel3d_b_engineering_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_engineering', id_only=True) vel3d_b_engineering_stream_def_id = self.pubsubclient.create_stream_definition( name='vel3d_b_engineering', parameter_dictionary_id=vel3d_b_engineering_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='vel3d_b_sample', description='vel3d_b_sample') data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=vel3d_b_sample_stream_def_id) self.damsclient.assign_data_product( input_resource_id=self.instrument_device_id, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) dp_obj = IonObject(RT.DataProduct, name='vel3d_b_engineering', description='vel3d_b_engineering') data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=vel3d_b_engineering_stream_def_id) self.damsclient.assign_data_product( input_resource_id=self.instrument_device_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') data_product_id3 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.damsclient.assign_data_product( input_resource_id=self.instrument_device_id, data_product_id=data_product_id3) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id3) #create instrument site and associated deployment site_object = IonObject(RT.InstrumentSite, name='InstrumentSite1') instrument_site_id = self.omsclient.create_instrument_site( instrument_site=site_object, parent_id=self.platform_site_id) log.debug('_create_instrument_site_and_deployment site id: %s', instrument_site_id) #create supporting objects for the Deployment resource # 1. temporal constraint # find current deployment using time constraints current_time = int(calendar.timegm(time.gmtime())) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) # 2. PlatformPort object which defines device to port map platform_port_obj = IonObject( OT.PlatformPort, reference_designator='GA01SUMO-FI003-03-CTDMO0999', port_type=PortTypeEnum.PAYLOAD, ip_address='0') # now create the Deployment deployment_obj = IonObject( RT.Deployment, name='TestInstrumentDeployment', description='some new deployment', context=IonObject(OT.CabledInstrumentDeploymentContext), constraint_list=[temporal_bounds], port_assignments={self.instrument_device_id: platform_port_obj}) instrument_deployment_id = self.omsclient.create_deployment( deployment=deployment_obj, site_id=instrument_site_id, device_id=self.instrument_device_id) log.debug('_create_instrument_site_and_deployment deployment_id: %s', instrument_deployment_id) def _start_port_agent(self, instrument_agent_instance_obj=None): """ Construct and start the port agent, ONLY NEEDED FOR INSTRUMENT AGENTS. """ _port_agent_config = instrument_agent_instance_obj.port_agent_config # It blocks until the port agent starts up or a timeout _pagent = PortAgentProcess.launch_process(_port_agent_config, test_mode=True) pid = _pagent.get_pid() port = _pagent.get_data_port() cmd_port = _pagent.get_command_port() log.info( "IMS:_start_pagent returned from PortAgentProcess.launch_process pid: %s ", pid) # Hack to get ready for DEMO. Further though needs to be put int # how we pass this config info around. host = 'localhost' driver_config = instrument_agent_instance_obj.driver_config comms_config = driver_config.get('comms_config') if comms_config: host = comms_config.get('addr') else: log.warn("No comms_config specified, using '%s'" % host) # Configure driver to use port agent port number. instrument_agent_instance_obj.driver_config['comms_config'] = { 'addr': host, 'cmd_port': cmd_port, 'port': port } instrument_agent_instance_obj.driver_config['pagent_pid'] = pid self.imsclient.update_instrument_agent_instance( instrument_agent_instance_obj) return self.imsclient.read_instrument_agent_instance( instrument_agent_instance_obj._id) def _start_platform(self): """ Starts the given platform waiting for it to transition to the UNINITIALIZED state (note that the agent starts in the LAUNCHING state). More in concrete the sequence of steps here are: - prepares subscriber to receive the UNINITIALIZED state transition - launches the platform process - waits for the start of the process - waits for the transition to the UNINITIALIZED state """ ############################################################## # prepare to receive the UNINITIALIZED state transition: async_res = AsyncResult() def consume_event(evt, *args, **kwargs): log.debug("Got ResourceAgentStateEvent %s from origin %r", evt.state, evt.origin) if evt.state == PlatformAgentState.UNINITIALIZED: async_res.set(evt) # start subscriber: sub = EventSubscriber(event_type="ResourceAgentStateEvent", origin=self.platform_device_id, callback=consume_event) sub.start() log.info( "registered event subscriber to wait for state=%r from origin %r", PlatformAgentState.UNINITIALIZED, self.platform_device_id) #self._event_subscribers.append(sub) sub._ready_event.wait(timeout=EVENT_TIMEOUT) ############################################################## # now start the platform: agent_instance_id = self.platform_agent_instance_id log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id) pid = self.imsclient.start_platform_agent_instance( platform_agent_instance_id=agent_instance_id) log.debug("start_platform_agent_instance returned pid=%s", pid) #wait for start agent_instance_obj = self.imsclient.read_platform_agent_instance( agent_instance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, self.platform_device_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (90), "The platform agent instance did not spawn in 90 seconds") # Start a resource agent client to talk with the agent. self._pa_client = ResourceAgentClient(self.platform_device_id, name=gate.process_id, process=FakeProcess()) log.debug("got platform agent client %s", str(self._pa_client)) ############################################################## # wait for the UNINITIALIZED event: async_res.get(timeout=self._receive_timeout) def _register_oms_listener(self): #load the paramaters and the param dicts necesssary for the VEL3D log.debug("---------- connect_to_oms ---------- ") log.debug("oms_uri = %s", OMS_URI) self.oms = CIOMSClientFactory.create_instance(OMS_URI) #buddha url url = "http://10.22.88.168:5000/ion-service/oms_event" log.info("test_oms_events_receive:setup http url %s", url) result = self.oms.event.register_event_listener(url) log.debug("_register_oms_listener register_event_listener result %s", result) #------------------------------------------------------------------------------------- # Set up the subscriber to catch the alert event #------------------------------------------------------------------------------------- def callback_for_alert(event, *args, **kwargs): log.debug("caught an OMSDeviceStatusEvent: %s", event) self.catch_alert.put(event) self.event_subscriber = EventSubscriber( event_type='OMSDeviceStatusEvent', callback=callback_for_alert) self.event_subscriber.start() self.addCleanup(self.event_subscriber.stop) result = self.oms.event.generate_test_event({ 'platform_id': 'fake_platform_id', 'message': "fake event triggered from CI using OMS' generate_test_event", 'severity': '3', 'group ': 'power' }) log.debug("_register_oms_listener generate_test_event result %s", result) def _stop_platform(self): try: self.IMS.stop_platform_agent_instance( self.platform_agent_instance_id) except Exception: log.warn( "platform_id=%r: Exception in IMS.stop_platform_agent_instance with " "platform_agent_instance_id = %r. Perhaps already dead.", self.platform_device_id, self.platform_agent_instance_id)
class VisStreamLauncher(ImmediateProcess): """ Class emulates a stream source from a NetCDF file. It emits a record of data every few seconds on a stream identified by a routing key. """ def on_init(self): log.debug("VizStreamProducer init. Self.id=%s" % self.id) def on_start(self): log.debug("VizStreamProducer start") self.data_source_name = self.CFG.get('name') self.dataset = self.CFG.get('dataset') # create a pubsub client and a resource registry client self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) # Dummy instrument related clients self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.IngestClient = IngestionManagementServiceClient(node=self.container.node) # create the pubsub client self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) # Additional code for creating a dummy instrument """ # Set up the preconditions. Look for an existing ingestion config while True: log.info("VisStreamLauncher:on_start: Waiting for an ingestion configuration to be available.") ingestion_cfgs, _ = self.rrclient.find_resources(RT.IngestionConfiguration, None, None, True) if len(ingestion_cfgs) > 0: break else: gevent.sleep(1) """ # Check to see if the data_product already exists in the system (for e.g re launching the code after a crash) dp_ids,_ = self.rrclient.find_resources(RT.DataProduct, None, self.data_source_name, True) if len(dp_ids) > 0: data_product_id = dp_ids[0] print '>>>>>>>>>>>>> Found dp_id = ', data_product_id else: # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name=self.data_source_name, description=self.data_source_name, model=self.data_source_name) instModel_id = self.imsclient.create_instrument_model(instModel_obj) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name=self.data_source_name, description=self.data_source_name, serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct,name=self.data_source_name,description='ctd stream test') data_product_id = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id, persist_data=True, persist_metadata=True) print '>>>>>>>>>>>> New dp_id = ', data_product_id # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id, PRED.hasStream, None, True) if self.dataset == 'sinusoidal': pid = self.container.spawn_process(name='ctd_test.' + self.data_source_name , module='ion.processes.data.sinusoidal_stream_publisher',cls='SinusoidalCtdPublisher',config={'process':{'stream_id':stream_ids[0]}}) else: pid = self.container.spawn_process(name='ctd_test.' + self.data_source_name , module='ion.processes.data.ctd_stream_publisher',cls='SimpleCtdPublisher',config={'process':{'stream_id':stream_ids[0]}}) def on_quit(self): log.debug("VizStreamProducer quit")
class TestPlatformInstrument(BaseIntTestPlatform): def setUp(self): self._start_container() self._pp = pprint.PrettyPrinter() log.debug("oms_uri = %s", OMS_URI) self.oms = CIOMSClientFactory.create_instance(OMS_URI) self._get_platform_attributes() url = OmsTestMixin.start_http_server() log.info("TestPlatformInstrument:setup http url %s", url) result = self.oms.event.register_event_listener(url) log.info( "TestPlatformInstrument:setup register_event_listener result %s", result) # response = self.oms.port.get_platform_ports('LPJBox_CI_Ben_Hall') # log.info("TestPlatformInstrument:setup get_platform_ports %s", response) self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.org_id = self.RR2.create(any_old(RT.Org)) log.debug("Org created: %s", self.org_id) # see _set_receive_timeout self._receive_timeout = 177 self.instrument_device = '' self.platform_device = '' self.platform_agent_instance_id = '' self._pa_client = '' def done(): CIOMSClientFactory.destroy_instance(self.oms) event_notifications = OmsTestMixin.stop_http_server() log.info("event_notifications = %s" % str(event_notifications)) self.addCleanup(done) def _get_platform_attributes(self): attr_infos = self.oms.attr.get_platform_attributes( 'LPJBox_CI_Ben_Hall') log.debug('_get_platform_attributes: %s', self._pp.pformat(attr_infos)) # ret_infos = attr_infos['LPJBox_CI_Ben_Hall'] # for attrName, attr_defn in ret_infos.iteritems(): # attr = AttrNode(attrName, attr_defn) # pnode.add_attribute(attr) return attr_infos @unittest.skip('Still in construction...') def test_platform_with_instrument_streaming(self): # # The following is with just a single platform and the single # instrument "SBE37_SIM_08", which corresponds to the one on port 4008. # #load the paramaters and the param dicts necesssary for the VEL3D self._load_params() #create the instrument device/agent/mode self._create_instrument_resources() #create the platform device, agent and instance self._create_platform_configuration('LPJBox_CI_Ben_Hall') self.rrclient.create_association(subject=self.platform_device, predicate=PRED.hasDevice, object=self.instrument_device) self._start_platform() # self.addCleanup(self._stop_platform, p_root) # get everything in command mode: self._ping_agent() self._initialize() _ia_client = ResourceAgentClient(self.instrument_device, process=FakeProcess()) state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) self._go_active() # self._run() gevent.sleep(3) # note that this includes the instrument also getting to the command state # self._stream_instruments() # get client to the instrument: # the i_obj is a DotDict with various pieces captured during the # set-up of the instrument, in particular instrument_device_id #i_obj = self._get_instrument(instr_key) # log.debug("KK creating ResourceAgentClient") # ia_client = ResourceAgentClient(i_obj.instrument_device_id, # process=FakeProcess()) # log.debug("KK got ResourceAgentClient: %s", ia_client) # # # verify the instrument is command state: # state = ia_client.get_agent_state() # log.debug("KK instrument state: %s", state) # self.assertEqual(state, ResourceAgentState.COMMAND) self._reset() self._shutdown() def _load_params(self): log.info( "--------------------------------------------------------------------------------------------------------" ) # load_parameter_scenarios self.container.spawn_process( "Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories= "ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) def _create_platform_configuration(self, platform_id, parent_platform_id=None): """ This method is an adaptation of test_agent_instance_config in test_instrument_management_service_integration.py @param platform_id @param parent_platform_id @return a DotDict with various of the constructed elements associated to the platform. """ tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() param_dict_name = 'platform_eng_parsed' parsed_rpdict_id = self.dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) self.parsed_stream_def_id = self.pubsubclient.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_rpdict_id) driver_config = PLTFRM_DVR_CONFIG driver_config['attributes'] = self._get_platform_attributes( ) #self._platform_attributes[platform_id] #OMS returning an error for port.get_platform_ports #driver_config['ports'] = self._platform_ports[platform_id] log.debug("driver_config: %s", driver_config) # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': driver_config}) platform_agent_instance_obj.agent_config = { 'platform_config': { 'platform_id': 'LPJBox_CI_Ben_Hall', 'parent_platform_id': None } } self.platform_agent_instance_id = self.imsclient.create_platform_agent_instance( platform_agent_instance_obj) # agent creation platform_agent_obj = any_old( RT.PlatformAgent, { "stream_configurations": self._get_platform_stream_configs(), 'driver_module': PLTFRM_DVR_MOD, 'driver_class': PLTFRM_DVR_CLS }) platform_agent_id = self.imsclient.create_platform_agent( platform_agent_obj) # device creation self.platform_device = self.imsclient.create_platform_device( any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, { "temporal_domain": tdom, "spatial_domain": sdom }) dp_id = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id) self.damsclient.assign_data_product( input_resource_id=self.platform_device, data_product_id=dp_id) self.dpclient.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.dpclient.delete_data_product, dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance( self.platform_agent_instance_id, self.platform_device) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition( platform_agent_id, self.platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource( self.platform_agent_instance_id, self.org_id) ####################################### # dataset log.debug('data product = %s', dp_id) stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) log.debug('Data product stream_ids = %s', stream_ids) stream_id = stream_ids[0] # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) ####################################### return def _create_instrument_resources(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='VEL3D', description="VEL3D") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') vel3d_b_sample = StreamConfiguration( stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample') vel3d_b_engineering = StreamConfiguration( stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering') # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri= "http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg", stream_configurations=[ raw_config, vel3d_b_sample, vel3d_b_engineering ]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name='VEL3DDevice', description="VEL3DDevice", serial_number="12345") self.instrument_device = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, self.instrument_device) port_agent_config = { 'device_addr': '10.180.80.6', 'device_port': 2101, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 1025, 'data_port': 1026, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='VEL3DAgentInstance', description="VEL3DAgentInstance", port_agent_config=port_agent_config, alerts=[]) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, self.instrument_device) self._start_port_agent( self.imsclient.read_instrument_agent_instance( instAgentInstance_id)) vel3d_b_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_sample', id_only=True) vel3d_b_sample_stream_def_id = self.pubsubclient.create_stream_definition( name='vel3d_b_sample', parameter_dictionary_id=vel3d_b_sample_pdict_id) vel3d_b_engineering_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'vel3d_b_engineering', id_only=True) vel3d_b_engineering_stream_def_id = self.pubsubclient.create_stream_definition( name='vel3d_b_engineering', parameter_dictionary_id=vel3d_b_engineering_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='vel3d_b_sample', description='vel3d_b_sample', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=vel3d_b_sample_stream_def_id) self.damsclient.assign_data_product( input_resource_id=self.instrument_device, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) dp_obj = IonObject(RT.DataProduct, name='vel3d_b_engineering', description='vel3d_b_engineering', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=vel3d_b_engineering_stream_def_id) self.damsclient.assign_data_product( input_resource_id=self.instrument_device, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id3 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.damsclient.assign_data_product( input_resource_id=self.instrument_device, data_product_id=data_product_id3) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id3) def _start_port_agent(self, instrument_agent_instance_obj=None): """ Construct and start the port agent, ONLY NEEDED FOR INSTRUMENT AGENTS. """ _port_agent_config = instrument_agent_instance_obj.port_agent_config # It blocks until the port agent starts up or a timeout _pagent = PortAgentProcess.launch_process(_port_agent_config, test_mode=True) pid = _pagent.get_pid() port = _pagent.get_data_port() cmd_port = _pagent.get_command_port() log.info( "IMS:_start_pagent returned from PortAgentProcess.launch_process pid: %s ", pid) # Hack to get ready for DEMO. Further though needs to be put int # how we pass this config info around. host = 'localhost' driver_config = instrument_agent_instance_obj.driver_config comms_config = driver_config.get('comms_config') if comms_config: host = comms_config.get('addr') else: log.warn("No comms_config specified, using '%s'" % host) # Configure driver to use port agent port number. instrument_agent_instance_obj.driver_config['comms_config'] = { 'addr': host, 'cmd_port': cmd_port, 'port': port } instrument_agent_instance_obj.driver_config['pagent_pid'] = pid self.imsclient.update_instrument_agent_instance( instrument_agent_instance_obj) return self.imsclient.read_instrument_agent_instance( instrument_agent_instance_obj._id) def _start_platform(self): """ Starts the given platform waiting for it to transition to the UNINITIALIZED state (note that the agent starts in the LAUNCHING state). More in concrete the sequence of steps here are: - prepares subscriber to receive the UNINITIALIZED state transition - launches the platform process - waits for the start of the process - waits for the transition to the UNINITIALIZED state """ ############################################################## # prepare to receive the UNINITIALIZED state transition: async_res = AsyncResult() def consume_event(evt, *args, **kwargs): log.debug("Got ResourceAgentStateEvent %s from origin %r", evt.state, evt.origin) if evt.state == PlatformAgentState.UNINITIALIZED: async_res.set(evt) # start subscriber: sub = EventSubscriber(event_type="ResourceAgentStateEvent", origin=self.platform_device, callback=consume_event) sub.start() log.info( "registered event subscriber to wait for state=%r from origin %r", PlatformAgentState.UNINITIALIZED, self.platform_device) #self._event_subscribers.append(sub) sub._ready_event.wait(timeout=EVENT_TIMEOUT) ############################################################## # now start the platform: agent_instance_id = self.platform_agent_instance_id log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id) pid = self.imsclient.start_platform_agent_instance( platform_agent_instance_id=agent_instance_id) log.debug("start_platform_agent_instance returned pid=%s", pid) #wait for start agent_instance_obj = self.imsclient.read_platform_agent_instance( agent_instance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, self.platform_device._id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (90), "The platform agent instance did not spawn in 90 seconds") # Start a resource agent client to talk with the agent. self._pa_client = ResourceAgentClient(self.platform_device, name=gate.process_id, process=FakeProcess()) log.debug("got platform agent client %s", str(self._pa_client)) ############################################################## # wait for the UNINITIALIZED event: async_res.get(timeout=self._receive_timeout)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] @unittest.skip("TBD") def test_activateInstrumentSample(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) print 'new InstrumentModel id = ', instModel_id # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent") try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) print 'new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") try: instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) driver_config = { 'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls': 'SBE37Driver', 'workdir': '/tmp/', } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config=driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<', '>>']) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubcli.create_stream_definition( container=ctd_stream_def) print 'new Stream Definition id = ', instDevice_id print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') try: data_product_id1 = self.dpclient.create_data_product( dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) print 'new dp_id = ', data_product_id1 self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) print 'Data product streams1 = ', stream_ids print 'Creating new RAW data product with a stream definition' raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubcli.create_stream_definition( container=raw_stream_def) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') try: data_product_id2 = self.dpclient.create_data_product( dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) print 'new dp_id = ', data_product_id2 self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) print 'Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. #self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess()) print 'activate_instrument: got ia client %s', self._ia_client log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) print retval log.debug("test_activateInstrumentSample: initialize %s", str(retval)) time.sleep(2) log.debug( "test_activateInstrumentSample: Sending go_active command (L4-CI-SA-RQ-334)" ) cmd = AgentCommand(command='go_active') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "test_activateInstrumentSample: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state)) cmd = AgentCommand(command='run') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentSample: calling acquire_sample ") cmd = AgentCommand(command='acquire_sample') reply = self._ia_client.execute(cmd) log.debug( "test_activateInstrumentSample: return from acquire_sample %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentSample: calling acquire_sample 2") cmd = AgentCommand(command='acquire_sample') reply = self._ia_client.execute(cmd) log.debug( "test_activateInstrumentSample: return from acquire_sample 2 %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentSample: calling acquire_sample 3") cmd = AgentCommand(command='acquire_sample') reply = self._ia_client.execute(cmd) log.debug( "test_activateInstrumentSample: return from acquire_sample 3 %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(2) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) def test_activateInstrumentStream(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) print 'new InstrumentModel id = ', instModel_id # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent") try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) print 'new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentStream: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") try: instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) log.debug( "test_activateInstrumentStream: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) driver_config = { 'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls': 'SBE37Driver', 'workdir': '/tmp/', } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config=driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<', '>>']) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubcli.create_stream_definition( container=ctd_stream_def) log.debug( 'test_activateInstrumentStream new Stream Definition id = %s', instDevice_id) log.debug( 'test_activateInstrumentStream Creating new CDM data product with a stream definition' ) dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') try: data_product_id1 = self.dpclient.create_data_product( dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) log.debug('test_activateInstrumentStream new dp_id = %s', str(data_product_id1)) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('test_activateInstrumentStream Data product streams1 = %s', str(stream_ids)) simdata_subscription_id = self.pubsubcli.create_subscription( query=StreamQuery([stream_ids[0]]), exchange_name='Sim_data_queue', name='SimDataSubscription', description='SimData SubscriptionDescription') def simdata_message_received(message, headers): input = str(message) log.debug("test_activateInstrumentStream: granule received: %s", input) subscriber_registrar = StreamSubscriberRegistrar( process=self.container, node=self.container.node) simdata_subscriber = subscriber_registrar.create_subscriber( exchange_name='Sim_data_queue', callback=simdata_message_received) # Start subscribers simdata_subscriber.start() # Activate subscriptions self.pubsubcli.activate_subscription(simdata_subscription_id) log.debug( 'test_activateInstrumentStream Creating new RAW data product with a stream definition' ) raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubcli.create_stream_definition( container=raw_stream_def) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') try: data_product_id2 = self.dpclient.create_data_product( dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) log.debug('test_activateInstrumentStream new dp_id = %s', str(data_product_id2)) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('test_activateInstrumentStream Data product streams2 = %s', str(stream_ids)) self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) log.debug( 'test_activateInstrumentStream Instrument agent instance obj: = %s', str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. #self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess()) log.debug('test_activateInstrumentStream: got ia client %s', str(self._ia_client)) log.debug("test_activateInstrumentStream: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: initialize %s", str(retval)) time.sleep(2) log.debug( "test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)" ) cmd = AgentCommand(command='go_active') reply = self._ia_client.execute_agent(cmd) log.debug( "test_activateInstrumentStream: return value from go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state)) cmd = AgentCommand(command='run') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: run %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentStream: calling go_streaming ") cmd = AgentCommand(command='go_streaming') reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentStream: return from go_streaming %s", str(reply)) time.sleep(15) log.debug("test_activateInstrumentStream: calling go_observatory") cmd = AgentCommand(command='go_observatory') reply = self._ia_client.execute(cmd) log.debug( "test_activateInstrumentStream: return from go_observatory %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentStream: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return from reset %s", str(reply)) time.sleep(2) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() #self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources( RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def cleanupprocs(self): stm = os.popen('ps -e | grep ion.agents.port.logger_process') procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) stm = os.popen('ps -e | grep ion.agents.instrument.zmq_driver_process') procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) # stm = os.popen('rm /tmp/*.pid.txt') @unittest.skip( "Deprecated by IngestionManagement refactor, timeout on start inst agent?" ) def test_deploy_activate_full(self): # ensure no processes or pids are left around by agents or Sims #self.cleanupprocs() self.loggerpids = [] #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD) try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) #------------------------------- # Create Instrument Site #------------------------------- instrumentSite_obj = IonObject(RT.InstrumentSite, name='instrumentSite1', description="SBE37IMInstrumentSite") try: instrumentSite_id = self.omsclient.create_instrument_site( instrument_site=instrumentSite_obj, parent_id='') except BadRequest as ex: self.fail("failed to create new InstrumentSite: %s" % ex) print 'test_deployAsPrimaryDevice: new instrumentSite id = ', instrumentSite_id self.omsclient.assign_instrument_model_to_instrument_site( instModel_id, instrumentSite_id) #------------------------------- # Logical Transform: Output Data Products #------------------------------- # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Old InstrumentDevice #------------------------------- instDevice_obj = IonObject( RT.InstrumentDevice, name='SBE37IMDeviceYear1', description="SBE37IMDevice for the FIRST year of deployment", serial_number="12345") try: oldInstDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, oldInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print 'test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ', oldInstDevice_id self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='SiteDataProduct', description='SiteDataProduct', temporal_domain=tdom, spatial_domain=sdom) instrument_site_output_dp_id = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=instrument_site_output_dp_id) #self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects( instrument_site_output_dp_id, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects( instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='first deployment') oldDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id) #------------------------------- # Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information # cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear1', description="SBE37IMAgentInstanceYear1", port_agent_config=port_agent_config, stream_configurations=[raw_config, parsed_config]) oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, oldInstDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() #------------------------------- # Create CTD Parsed as the Year 1 data product and attach to instrument #------------------------------- print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year1', description='ctd stream test year 1', temporal_domain=tdom, spatial_domain=sdom) ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year1 self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects( ctd_parsed_data_product_year1, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams1 = ', stream_ids #------------------------------- # Create New InstrumentDevice #------------------------------- instDevice_obj_2 = IonObject( RT.InstrumentDevice, name='SBE37IMDeviceYear2', description="SBE37IMDevice for the SECOND year of deployment", serial_number="67890") try: newInstDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj_2) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, newInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print 'test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ', newInstDevice_id #set the LCSTATE self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE) instDevice_obj_2 = self.rrclient.read(newInstDevice_id) log.debug( "test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate)) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='second deployment') newDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id) #------------------------------- # Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4004, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 4005, 'data_port': 4006, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear2', description="SBE37IMAgentInstanceYear2", port_agent_config=port_agent_config) newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, newInstDevice_id) #------------------------------- # Create CTD Parsed as the Year 2 data product #------------------------------- dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year2', description='ctd stream test year 2', temporal_domain=tdom, spatial_domain=sdom) ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year2 self.damsclient.assign_data_product( input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects( ctd_parsed_data_product_year2, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug( "test_deployAsPrimaryDevice: create data process definition ctd_L0_all" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new ctd_L0_all data process definition: %s" % ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name='L0_Conductivity', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity') outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name='L0_Pressure', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure') outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name='L0_Temperature', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature') self.out_prod_dict = {} log.debug( "test_deployAsPrimaryDevice: create output data product L0 conductivity" ) ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_conductivity_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.out_prod_dict['conductivity'] = ctd_l0_conductivity_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug( "test_deployAsPrimaryDevice: create output data product L0 pressure" ) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_pressure_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.out_prod_dict['pressure'] = ctd_l0_pressure_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug( "test_deployAsPrimaryDevice: create output data product L0 temperature" ) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_temperature_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.out_prod_dict['temperature'] = ctd_l0_temperature_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product) #------------------------------- log.debug( "test_deployAsPrimaryDevice: create L0 all data_process start") try: out_data_products = self.out_prod_dict.values() ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], out_data_products) self.dataprocessclient.activate_data_process( ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "test_deployAsPrimaryDevice: create L0 all data_process return") #-------------------------------- # Activate the deployment #-------------------------------- self.omsclient.activate_deployment(oldDeployment_id) #------------------------------- # Launch InstrumentAgentInstance Sim1, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=oldInstAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=oldInstAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance( oldInstAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, oldInstDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance( oldInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent1_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim1 = ResourceAgentClient('iaclient Sim1', name=gate.process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim1 %s', self._ia_client_sim1 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1)) #------------------------------- # Launch InstrumentAgentInstance Sim2, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=newInstAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=newInstAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance( newInstAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, oldInstDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance( newInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent2_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim2 = ResourceAgentClient('iaclient Sim2', name=gate.process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim2 %s', self._ia_client_sim2 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2)) #------------------------------- # Streaming Sim1 (old instrument) #------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active %s", str(reply)) self.assertTrue(reply) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim1.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from START_AUTOSAMPLE: %s", str(retval)) #------------------------------- # Streaming Sim 2 (new instrument) #------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim2.execute_agent(cmd) log.debug( "test_deployAsPrimaryDevice: return value from go_active_sim2 %s", str(reply)) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim2.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s", str(retval)) gevent.sleep(10) #------------------------------- # Shutdown Sim1 (old instrument) #------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(5) #------------------------------- # Shutdown Sim2 (old instrument) #------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug( "test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset_sim2 ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset_sim2 %s", str(reply)) time.sleep(5)
class TestInstrumentAlerts(IonIntegrationTestCase): pdict_id = None def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.catch_alert = gevent.queue.Queue() def _create_instrument_model(self): instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) self.addCleanup(self.imsclient.delete_instrument_model, instModel_id) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.addCleanup(self.imsclient.delete_instrument_agent, instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) self.addCleanup(self.imsclient.delete_instrument_device, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_stream_alarms(self, instDevice_id): #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ temp_alert_def = { 'name': 'temperature_warning_interval', 'stream_name': 'parsed', 'description': 'Temperature is below the normal range of 50.0 and above.', 'alert_type': StreamAlertType.WARNING, 'aggregate_type': AggregateStatusType.AGGREGATE_DATA, 'value_id': 'temp', 'resource_id': instDevice_id, 'origin_type': 'device', 'lower_bound': 50.0, 'lower_rel_op': '<', 'alert_class': 'IntervalAlert' } late_data_alert_def = { 'name': 'late_data_warning', 'stream_name': 'parsed', 'description': 'Expected data has not arrived.', 'alert_type': StreamAlertType.WARNING, 'aggregate_type': AggregateStatusType.AGGREGATE_COMMS, 'value_id': None, 'resource_id': instDevice_id, 'origin_type': 'device', 'time_delta': 2, 'alert_class': 'LateDataAlert' } return temp_alert_def, late_data_alert_def def _create_instrument_agent_instance(self, instAgent_id, instDevice_id): # port_agent_config = { # 'device_addr': CFG.device.sbe37.host, # 'device_port': CFG.device.sbe37.port, # 'process_type': PortAgentProcessType.UNIX, # 'binary_path': "port_agent", # 'port_agent_addr': 'localhost', # 'command_port': CFG.device.sbe37.port_agent_cmd_port, # 'data_port': CFG.device.sbe37.port_agent_data_port, # 'log_level': 5, # 'type': PortAgentType.ETHERNET # } port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': 4008, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } temp_alert, late_data_alert = self._create_instrument_stream_alarms( instDevice_id) instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config, alerts=[temp_alert, late_data_alert]) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) self.addCleanup(self.imsclient.delete_instrument_agent_instance, instAgentInstance_id) return instAgentInstance_id def test_alerts(self): #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) # It is necessary for the instrument device to be associated with atleast one output data product tdom, sdom = time_series_domain() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) # We are creating two data products here, one for parsed and another raw dp_obj_parsed = IonObject( RT.DataProduct, name='parsed_data_product', description='Parsed output data product for instrument', temporal_domain=tdom.dump(), spatial_domain=sdom.dump()) dp_obj_raw = IonObject( RT.DataProduct, name='raw_data_prod', description='Raw output data product for instrument', temporal_domain=tdom.dump(), spatial_domain=sdom.dump()) parsed_out_data_prod_id = self.dataproductclient.create_data_product( data_product=dp_obj_parsed, stream_definition_id=parsed_stream_def_id) raw_out_data_prod_id = self.dataproductclient.create_data_product( data_product=dp_obj_raw, stream_definition_id=raw_stream_def_id) self.addCleanup(self.dataproductclient.delete_data_product, parsed_out_data_prod_id) self.addCleanup(self.dataproductclient.delete_data_product, raw_out_data_prod_id) self.dataproductclient.activate_data_product_persistence( data_product_id=parsed_out_data_prod_id) self.dataproductclient.activate_data_product_persistence( data_product_id=raw_out_data_prod_id) # todo: note that the generated config on the instruments will be done for both raw and parsed stream defs since these two data products constructed with each are associated as output data products with the instrument # todo: if the config is not generated for a stream def, then the instrument agent will complain if the simulator generates data corresponding to a stream def that is not there in the stream config as a mentioned stream def self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=parsed_out_data_prod_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=raw_out_data_prod_id) log.debug("assigned instdevice id: %s to data product: %s", instDevice_id, raw_out_data_prod_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance( instAgent_id, instDevice_id) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) # Wait for instrument agent to spawn gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Set up the subscriber to catch the alert event #------------------------------------------------------------------------------------- def callback_for_alert(event, *args, **kwargs): log.debug("caught an alert: %s", event) self.catch_alert.put(event) self.event_subscriber = EventSubscriber( event_type='DeviceStatusAlertEvent', origin=instDevice_id, callback=callback_for_alert) self.event_subscriber.start() self.addCleanup(self.event_subscriber.stop) #------------------------------------------------------------------------------------- # Running the instrument.... #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) got_bad_temp = False got_late_data = False runtime = 0 starttime = time.time() caught_events = [] while (got_bad_temp == False or got_late_data == False) and \ runtime < 120: a = self.catch_alert.get(timeout=90) caught_events.append(a) if a.name == 'temperature_warning_interval' and \ a.description == 'Alert triggered by out of range data values: temp ': got_bad_temp = True if a.name == 'late_data_warning' and \ a.description == 'Expected data has not arrived.': got_late_data = True runtime = time.time() - starttime log.debug("caught_events: %s", [c.name for c in caught_events]) for c in caught_events: self.assertIn( c.name, ['temperature_warning_interval', 'late_data_warning']) self.assertEqual(c.origin, instDevice_id) self.assertEqual(c.type_, 'DeviceStatusAlertEvent') self.assertEqual(c.origin_type, 'InstrumentDevice') self.assertTrue(got_bad_temp) self.assertTrue(got_late_data)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) print 'started services' # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(StatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config(self): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) iconfig_builder = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_id = self.RR2.create(any_old(RT.Org)) inst_startup_config = {'startup': 'config'} required_config_keys = [ 'org_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'alarm_defs', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('stream_config', config) if None is inst_device_id: for key in ['alarm_defs', 'children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure() platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj) child_config = pconfig_builder.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure() platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id) child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) def sample_nested_platform_agent_instance_config(self): """ for informational purposes """ ret = {'org_name': 'Org_1', 'alarm_defs': {}, 'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)}, 'stream_config': {'parameter_dictionary': 'lots of stuff'}, 'agent': {'resource_id': '33e54106c4444444862da082098bc123'}, 'startup_config': {}, 'device_type': 'PlatformDevice', 'children': {'76a39596eeff4fd5b409c4cb93f0e581': {'org_name': 'Org_1', 'alarm_defs': {}, 'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)}, 'stream_config': {'parameter_dictionary': 'lots of stuff'}, 'agent': {'resource_id': '76a39596eeff4fd5b409c4cb93f0e581'}, 'startup_config': {}, 'device_type': 'PlatformDevice', 'children': {}}}} return ret
class TestCTDTransformsIntegration(IonIntegrationTestCase): pdict_id = None def setUp(self): # Start container # print 'instantiating container' self._start_container() # container = Container() # print 'starting container' # container.start() # print 'started container' self.container.start_rel_from_url("res/deploy/r2deploy.yml") print "started services" # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataset_management = self.datasetclient def create_logger(self, name, stream_id=""): # logger process producer_definition = ProcessDefinition(name=name + "_logger") producer_definition.executable = { "module": "ion.processes.data.stream_granule_logger", "class": "StreamGranuleLogger", } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = {"process": {"stream_id": stream_id}} pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration ) return pid def _create_instrument_model(self): instModel_obj = IonObject( RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", stream_configuration={"raw": "ctd_raw_param_dict", "parsed": "ctd_parsed_param_dict"}, ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) return instModel_id def _create_instrument_agent(self, instModel_id): instAgent_obj = IonObject( RT.InstrumentAgent, name="agent007", description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", ) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject( RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id ) return instDevice_id def _create_instrument_agent_instance(self, instAgent_id, instDevice_id): port_agent_config = { "device_addr": "sbe37-simulator.oceanobservatories.org", "device_port": 4001, "process_type": PortAgentProcessType.UNIX, "binary_path": "port_agent", "command_port": 4003, "data_port": 4000, "log_level": 5, } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstance", description="SBE37IMAgentInstance", port_agent_config=port_agent_config, ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id ) return instAgentInstance_id def _create_param_dicts(self): tdom, sdom = time_series_domain() self.sdom = sdom.dump() self.tdom = tdom.dump() self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) def _create_input_data_products(self, ctd_stream_def_id, instDevice_id): dp_obj = IonObject( RT.DataProduct, name="the parsed data", description="ctd stream test", temporal_domain=self.tdom, spatial_domain=self.sdom, ) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) # --------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product # --------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) pid = self.create_logger("ctd_parsed", stream_ids[0]) self.loggerpids.append(pid) # --------------------------------------------------------------------------- # Create CTD Raw as the second data product # --------------------------------------------------------------------------- if not self.pdict_id: self._create_param_dicts() raw_stream_def_id = self.pubsubclient.create_stream_definition( name="SBE37_RAW", parameter_dictionary_id=self.pdict_id ) dp_obj = IonObject( RT.DataProduct, name="the raw data", description="raw stream test", temporal_domain=self.tdom, spatial_domain=self.sdom, ) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product) # --------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product # --------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) # --------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product # --------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print "Data product streams2 = ", stream_ids return ctd_parsed_data_product def _create_data_process_definitions(self): # ------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition # ------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L0_all", description="transform ctd package into three separate L0 streams", module="ion.processes.data.transforms.ctd.ctd_L0_all", class_name="ctd_L0_all", ) self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # ------------------------------------------------------------------------------------- # L1 Conductivity: Data Process Definition # ------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L1_conductivity", description="create the L1 conductivity data product", module="ion.processes.data.transforms.ctd.ctd_L1_conductivity", class_name="CTDL1ConductivityTransform", ) self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # ------------------------------------------------------------------------------------- # L1 Pressure: Data Process Definition # ------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L1_pressure", description="create the L1 pressure data product", module="ion.processes.data.transforms.ctd.ctd_L1_pressure", class_name="CTDL1PressureTransform", ) self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # ------------------------------------------------------------------------------------- # L1 Temperature: Data Process Definition # ------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L1_temperature", description="create the L1 temperature data product", module="ion.processes.data.transforms.ctd.ctd_L1_temperature", class_name="CTDL1TemperatureTransform", ) self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # ------------------------------------------------------------------------------------- # L2 Salinity: Data Process Definition # ------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L2_salinity", description="create the L1 temperature data product", module="ion.processes.data.transforms.ctd.ctd_L2_salinity", class_name="SalinityTransform", ) self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # ------------------------------------------------------------------------------------- # L2 Density: Data Process Definition # ------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L2_density", description="create the L1 temperature data product", module="ion.processes.data.transforms.ctd.ctd_L2_density", class_name="DensityTransform", ) self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) return ( self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id, self.ctd_L1_pressure_dprocdef_id, self.ctd_L1_temperature_dprocdef_id, self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id, ) def _create_stream_definitions(self): if not self.pdict_id: self._create_param_dicts() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name="L0_Conductivity", parameter_dictionary_id=self.pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding="conductivity" ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name="L0_Pressure", parameter_dictionary_id=self.pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding="pressure" ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name="L0_Temperature", parameter_dictionary_id=self.pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding="temperature" ) return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id def _create_l0_output_data_products( self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id ): output_products = {} ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name="L0_Conductivity", description="transform output conductivity", temporal_domain=self.tdom, spatial_domain=self.sdom, ) self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id ) output_products["conductivity"] = self.ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_conductivity_output_dp_id) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name="L0_Pressure", description="transform output pressure", temporal_domain=self.tdom, spatial_domain=self.sdom, ) self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id ) output_products["pressure"] = self.ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_pressure_output_dp_id) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name="L0_Temperature", description="transform output temperature", temporal_domain=self.tdom, spatial_domain=self.sdom, ) self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id ) output_products["temperature"] = self.ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_temperature_output_dp_id) return output_products def _create_l1_out_data_products(self): ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name="L1_Conductivity", description="transform output L1 conductivity", temporal_domain=self.tdom, spatial_domain=self.sdom, ) self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id ) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger("ctd_l1_conductivity", stream_ids[0]) self.loggerpids.append(pid) ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name="L1_Pressure", description="transform output L1 pressure", temporal_domain=self.tdom, spatial_domain=self.sdom, ) self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id ) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger("ctd_l1_pressure", stream_ids[0]) self.loggerpids.append(pid) ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name="L1_Temperature", description="transform output L1 temperature", temporal_domain=self.tdom, spatial_domain=self.sdom, ) self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id ) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger("ctd_l1_temperature", stream_ids[0]) self.loggerpids.append(pid) def _create_l2_out_data_products(self): # ------------------------------- # L2 Salinity - Density: Output Data Products # ------------------------------- if not self.pdict_id: self._create_param_dicts() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition( name="L2_salinity", parameter_dictionary_id=self.pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding="salinity" ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition( name="L2_Density", parameter_dictionary_id=self.pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding="density" ) ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name="L2_Salinity", description="transform output L2 salinity", temporal_domain=self.tdom, spatial_domain=self.sdom, ) self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id ) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger("ctd_l2_salinity", stream_ids[0]) self.loggerpids.append(pid) ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name="L2_Density", description="transform output pressure", temporal_domain=self.tdom, spatial_domain=self.sdom, ) self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id ) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger("ctd_l2_density", stream_ids[0]) self.loggerpids.append(pid) def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] # ------------------------------------------------------------------------------------- # Create InstrumentModel # ------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() # ------------------------------------------------------------------------------------- # Create InstrumentAgent # ------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) # ------------------------------------------------------------------------------------- # Create InstrumentDevice # ------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) # ------------------------------------------------------------------------------------- # Create Instrument Agent Instance # ------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id, instDevice_id) # ------------------------------------------------------------------------------------- # create a stream definition for the data from the ctd simulator # ------------------------------------------------------------------------------------- self._create_param_dicts() ctd_stream_def_id = self.pubsubclient.create_stream_definition( name="SBE37_CDM", parameter_dictionary_id=self.pdict_id ) # ------------------------------------------------------------------------------------- # Create two data products # ------------------------------------------------------------------------------------- ctd_parsed_data_product = self._create_input_data_products(ctd_stream_def_id, instDevice_id) # ------------------------------------------------------------------------------------- # Create data process definitions # ------------------------------------------------------------------------------------- self._create_data_process_definitions() # ------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products # ------------------------------------------------------------------------------------- outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id = ( self._create_stream_definitions() ) self.output_products = {} self.output_products = self._create_l0_output_data_products( outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id ) self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition( name="L1_conductivity", parameter_dictionary_id=self.pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding="conductivity" ) self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition( name="L1_Pressure", parameter_dictionary_id=self.pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding="pressure" ) self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition( name="L1_Temperature", parameter_dictionary_id=self.pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding="temperature" ) self._create_l1_out_data_products() self._create_l2_out_data_products() # ------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process # ------------------------------------------------------------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products ) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) # ------------------------------------------------------------------------------------- # L1 Conductivity: Create the data process # ------------------------------------------------------------------------------------- l1_conductivity_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_conductivity_dprocdef_id, [self.ctd_l0_conductivity_output_dp_id], {"conductivity": self.ctd_l1_conductivity_output_dp_id}, ) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) # ------------------------------------------------------------------------------------- # L1 Pressure: Create the data process # ------------------------------------------------------------------------------------- l1_pressure_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_pressure_dprocdef_id, [self.ctd_l0_pressure_output_dp_id], {"pressure": self.ctd_l1_pressure_output_dp_id}, ) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) # ------------------------------------------------------------------------------------- # L1 Temperature: Create the data process # ------------------------------------------------------------------------------------- l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_temperature_dprocdef_id, [self.ctd_l0_temperature_output_dp_id], {"temperature": self.ctd_l1_temperature_output_dp_id}, ) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) # ------------------------------------------------------------------------------------- # L2 Salinity: Create the data process # ------------------------------------------------------------------------------------- l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {"salinity": self.ctd_l2_salinity_output_dp_id} ) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) # ------------------------------------------------------------------------------------- # L2 Density: Create the data process # ------------------------------------------------------------------------------------- l2_density_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {"density": self.ctd_l2_density_output_dp_id} ) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) # ------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client # ------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup( self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id ) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) # Wait for instrument agent to spawn gate = ProcessStateGate( self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING ) self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient( instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess() ) # ------------------------------------------------------------------------------------- # Streaming # ------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, "DRIVER_STATE_COMMAND") cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) # todo ResourceAgentClient no longer has method set_param # # Make sure the sampling rate and transmission are sane. # params = { # SBE37Parameter.NAVG : 1, # SBE37Parameter.INTERVAL : 5, # SBE37Parameter.TXREALTIME : True # } # self._ia_client.set_param(params) # todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it? cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) gevent.sleep(15) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it? # log.debug("test_activateInstrumentStream: calling go_observatory") # cmd = AgentCommand(command='go_observatory') # reply = self._ia_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) # ------------------------------------------------------------------------------------------------- # Cleanup processes # ------------------------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node) def test_createDataProcess(self): #--------------------------------------------------------------------------- # Data Process Definition #--------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # Make assertion on the newly registered data process definition data_process_definition = self.rrclient.read(dprocdef_id) self.assertEquals(data_process_definition.name, 'ctd_L0_all') self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams') self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all') self.assertEquals(data_process_definition.class_name, 'ctd_L0_all') # Read the data process definition using data process management and make assertions dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id) self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all') self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all') #--------------------------------------------------------------------------- # Create an input instrument #--------------------------------------------------------------------------- instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) # Register the instrument so that the data producer and stream object are created data_producer_id = self.damsclient.register_instrument(instrument_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id ) # Assert that the link between the stream definition and the data process definition was done assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True) self.assertIsNotNone(assocs) #--------------------------------------------------------------------------- # Input Data Product #--------------------------------------------------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() input_dp_obj = IonObject( RT.DataProduct, name='InputDataProduct', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test') #Make assertions on the input data product created input_dp_obj = self.rrclient.read(input_dp_id) self.assertEquals(input_dp_obj.name, 'InputDataProduct') self.assertEquals(input_dp_obj.description, 'some new dp') self.damsclient.assign_data_product(instrument_id, input_dp_id) # Retrieve the stream via the DataProduct->Stream associations stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True) self.in_stream_id = stream_ids[0] #--------------------------------------------------------------------------- # Output Data Product #--------------------------------------------------------------------------- outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' ) outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' ) outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' ) self.output_products={} output_dp_obj = IonObject(RT.DataProduct, name='conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id) self.output_products['conductivity'] = output_dp_id_1 output_dp_obj = IonObject(RT.DataProduct, name='pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id) self.output_products['pressure'] = output_dp_id_2 output_dp_obj = IonObject(RT.DataProduct, name='temperature', description='transform output ', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id) self.output_products['temperature'] = output_dp_id_3 #--------------------------------------------------------------------------- # Create the data process #--------------------------------------------------------------------------- def _create_data_process(): dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products) return dproc_id dproc_id = _create_data_process() # Make assertions on the data process created data_process = self.dataprocessclient.read_data_process(dproc_id) # Assert that the data process has a process id attached self.assertIsNotNone(data_process.process_id) # Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute self.assertIsNotNone(data_process.input_subscription_id) output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True) self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3])) @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}}) def test_createDataProcessUsingSim(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg") instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) #------------------------------- # Create CTD Raw as the second data product #------------------------------- raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj.name = 'ctd_raw' ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # Create listener for data process events and verify that events are received. #------------------------------- # todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events #todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_id = data_process.process_id self.addCleanup(self.process_dispatcher.cancel_process, process_id) #------------------------------- # Wait until the process launched in the create_data_process() method is actually running, before proceeding further in this test #------------------------------- gate = ProcessStateGate(self.process_dispatcher.read_process, process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The data process (%s) did not spawn in 30 seconds" % process_id) #------------------------------- # Retrieve a list of all data process defintions in RR and validate that the DPD is listed #------------------------------- # todo: Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions # todo: data topics are being handled by pub sub at the level of streams self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) #todo: check that activate event is received L4-CI-SA-RQ-367 #todo... (it looks like no event is being published when the data process is activated... so below, we just check for now # todo... that the subscription is indeed activated) (Swarbhanu) # todo: monitor process to see if it is active (sa-rq-182) ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id) input_subscription_id = ctd_l0_all_data_process.input_subscription_id subs = self.rrclient.read(input_subscription_id) self.assertTrue(subs.activated) # todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the extended resources for data process definition and for data process #------------------------------- extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) self.assertEqual(1, len(extended_process_definition.data_processes)) log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition)) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEqual(1, len(extended_process.input_data_products)) log.debug("test_createDataProcess: extended_process %s", str(extended_process)) ################################ Test the removal of data processes ################################## #------------------------------------------------------------------- # Test the deleting of the data process #------------------------------------------------------------------- # Before deleting, get the input streams, output streams and the subscriptions so that they can be checked after deleting # dp_obj_1 = self.rrclient.read(ctd_l0_all_data_process_id) # input_subscription_id = dp_obj_1.input_subscription_id # out_prods, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) # in_prods, _ = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasInputProduct, id_only=True) # in_streams = [] # for in_prod in in_prods: # streams, _ = self.rrclient.find_objects(in_prod, PRED.hasStream, id_only=True) # in_streams.extend(streams) # out_streams = [] # for out_prod in out_prods: # streams, _ = self.rrclient.find_objects(out_prod, PRED.hasStream, id_only=True) # out_streams.extend(streams) # Deleting the data process self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id) # Check that the data process got removed. Check the lcs state. It should be retired dp_obj = self.rrclient.read(ctd_l0_all_data_process_id) self.assertEquals(dp_obj.lcstate, LCS.RETIRED) # Check for process defs still attached to the data process dpd_assn_ids = self.rrclient.find_associations(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcessDefinition, id_only=True) self.assertEquals(len(dpd_assn_ids), 0) # Check for output data product still attached to the data process out_products, assocs = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) self.assertEquals(len(out_products), 0) self.assertEquals(len(assocs), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check of the data process has been deactivated self.assertIsNone(dp_obj.input_subscription_id) # Read the original subscription id of the data process and check that it has been deactivated with self.assertRaises(NotFound): self.pubsubclient.read_subscription(input_subscription_id) #------------------------------------------------------------------- # Delete the data process definition #------------------------------------------------------------------- # before deleting, get the process definition being associated to in order to be able to check later if the latter gets deleted as it should proc_def_ids, proc_def_asocs = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasProcessDefinition) self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id) # check that the data process definition has been retired dp_proc_def = self.rrclient.read(ctd_L0_all_dprocdef_id) self.assertEquals(dp_proc_def.lcstate, LCS.RETIRED) # Check for old associations of this data process definition proc_defs, proc_def_asocs = self.rrclient.find_objects(ctd_L0_all_dprocdef_id, PRED.hasProcessDefinition) self.assertEquals(len(proc_defs), 0) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject= ctd_L0_all_dprocdef_id, id_only=True) self.assertEquals(len(obj_assns), 0) ################################ Test the removal of data processes ################################## # Try force delete... This should simply delete the associations and the data process object # from the resource registry #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id) #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process definition #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id) def test_transform_function_crd(self): tf = TransformFunction(name='simple', module='pyon.ion.process', cls='SimpleProcess') tf_id = self.dataprocessclient.create_transform_function(tf) self.assertTrue(tf_id) tf2_id = self.dataprocessclient.create_transform_function(tf) self.assertEquals(tf_id, tf2_id) tf_obj = self.dataprocessclient.read_transform_function(tf_id) self.assertEquals([tf.name, tf.module, tf.cls, tf.function_type], [tf_obj.name, tf_obj.module, tf_obj.cls, tf_obj.function_type]) tf.module = 'dev.null' self.assertRaises(BadRequest, self.dataprocessclient.create_transform_function, tf) self.dataprocessclient.delete_transform_function(tf_id) self.assertRaises(NotFound, self.dataprocessclient.read_transform_function, tf_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Lifecycle self.assertEquals(len(extended_instrument.lcstate_transitions), 7) self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce'])) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) self.assertEquals(extended_platform.platform_agent._id, platform_agent_id) self.assertEquals(len(extended_platform.lcstate_transitions), 7) self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce'])) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(StatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config(self): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) iconfig_builder = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = {'lvl1': {'lvl2': 'lvl3val'}} required_config_keys = [ 'org_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alert_config', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.name, config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('aparam_alert_config', config) self.assertEqual(generic_alerts_config, config['aparam_alert_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'}, 'alerts': generic_alerts_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config, 'alerts': generic_alerts_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure() platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj) child_config = pconfig_builder.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure() platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id) child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.org_management_service = OrgManagementServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #print 'TestObservatoryManagementServiceIntegration: started services' self.event_publisher = EventPublisher() # @unittest.skip('this exists only for debugging the launch process') # def test_just_the_setup(self): # return def destroy(self, resource_ids): self.OMS.force_delete_observatory(resource_ids.observatory_id) self.OMS.force_delete_subsite(resource_ids.subsite_id) self.OMS.force_delete_subsite(resource_ids.subsite2_id) self.OMS.force_delete_subsite(resource_ids.subsiteb_id) self.OMS.force_delete_subsite(resource_ids.subsitez_id) self.OMS.force_delete_platform_site(resource_ids.platform_site_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id) self.OMS.force_delete_platform_site(resource_ids.platform_site3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_siteb3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id) #@unittest.skip('targeting') def test_resources_associations(self): resources = self._make_associations() self.destroy(resources) #@unittest.skip('targeting') def test_find_related_frames_of_reference(self): # finding subordinates gives a dict of obj lists, convert objs to ids def idify(adict): ids = {} for k, v in adict.iteritems(): ids[k] = [] for obj in v: ids[k].append(obj._id) return ids # a short version of the function we're testing, with id-ify def short(resource_id, output_types): ret = self.OMS.find_related_frames_of_reference(resource_id, output_types) return idify(ret) #set up associations first stuff = self._make_associations() #basic traversal of tree from instrument to platform ids = short(stuff.instrument_site_id, [RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) #since this is the first search, just make sure the input inst_id got stripped if RT.InstrumentSite in ids: self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #basic traversal of tree from platform to instrument ids = short(stuff.platform_siteb_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite]) #full traversal of tree from observatory down to instrument ids = short(stuff.observatory_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #full traversal of tree from instrument to observatory ids = short(stuff.instrument_site_id, [RT.Observatory]) self.assertIn(RT.Observatory, ids) self.assertIn(stuff.observatory_id, ids[RT.Observatory]) #partial traversal, only down to platform ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(RT.InstrumentSite, ids) #partial traversal, only down to platform ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertNotIn(RT.Observatory, ids) self.destroy(stuff) def _make_associations(self): """ create one of each resource and association used by OMS to guard against problems in ion-definitions """ #raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41") """ the tree we're creating (observatory, sites, platforms, instruments) rows are lettered, colums numbered. - first row is implied a - first column is implied 1 - site Z, just because O--Sz | S--S2--P3--I4 | Sb-Pb2-Ib3 | P--I2 <- PlatformDevice, InstrumentDevice2 | Pb <- PlatformDevice b | I <- InstrumentDevice """ org_id = self.OMS.create_marine_facility(any_old(RT.Org)) def create_under_org(resource_type): obj = any_old(resource_type) if RT.InstrumentDevice == resource_type: resource_id = self.IMS.create_instrument_device(obj) else: resource_id, _ = self.RR.create(obj) self.OMS.assign_resource_to_observatory_org(resource_id=resource_id, org_id=org_id) return resource_id #stuff we control observatory_id = create_under_org(RT.Observatory) subsite_id = create_under_org(RT.Subsite) subsite2_id = create_under_org(RT.Subsite) subsiteb_id = create_under_org(RT.Subsite) subsitez_id = create_under_org(RT.Subsite) platform_site_id = create_under_org(RT.PlatformSite) platform_siteb_id = create_under_org(RT.PlatformSite) platform_siteb2_id = create_under_org(RT.PlatformSite) platform_site3_id = create_under_org(RT.PlatformSite) instrument_site_id = create_under_org(RT.InstrumentSite) instrument_site2_id = create_under_org(RT.InstrumentSite) instrument_siteb3_id = create_under_org(RT.InstrumentSite) instrument_site4_id = create_under_org(RT.InstrumentSite) #stuff we associate to instrument_device_id = create_under_org(RT.InstrumentDevice) instrument_device2_id = create_under_org(RT.InstrumentDevice) platform_device_id = create_under_org(RT.PlatformDevice) platform_deviceb_id = create_under_org(RT.PlatformDevice) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) deployment_id, _ = self.RR.create(any_old(RT.Deployment)) #observatory self.RR.create_association(observatory_id, PRED.hasSite, subsite_id) self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id) #site self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id) self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id) self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id) #platform_site(s) self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id) self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id) self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id) self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id) self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id) self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id) #instrument_site(s) self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id) self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) ret = DotDict() ret.org_id = org_id ret.observatory_id = observatory_id ret.subsite_id = subsite_id ret.subsite2_id = subsite2_id ret.subsiteb_id = subsiteb_id ret.subsitez_id = subsitez_id ret.platform_site_id = platform_site_id ret.platform_siteb_id = platform_siteb_id ret.platform_siteb2_id = platform_siteb2_id ret.platform_site3_id = platform_site3_id ret.instrument_site_id = instrument_site_id ret.instrument_site2_id = instrument_site2_id ret.instrument_siteb3_id = instrument_siteb3_id ret.instrument_site4_id = instrument_site4_id ret.instrument_device_id = instrument_device_id ret.instrument_device2_id = instrument_device2_id ret.platform_device_id = platform_device_id ret.platform_deviceb_id = platform_deviceb_id ret.instrument_model_id = instrument_model_id ret.platform_model_id = platform_model_id ret.deployment_id = deployment_id return ret #@unittest.skip("targeting") def test_create_observatory(self): observatory_obj = IonObject(RT.Observatory, name='TestFacility', description='some new mf') observatory_id = self.OMS.create_observatory(observatory_obj) self.OMS.force_delete_observatory(observatory_id) #@unittest.skip("targeting") def test_find_observatory_org(self): org_obj = IonObject(RT.Org, name='TestOrg', description='some new mf org') org_id = self.OMS.create_marine_facility(org_obj) observatory_obj = IonObject(RT.Observatory, name='TestObservatory', description='some new obs') observatory_id = self.OMS.create_observatory(observatory_obj) #make association self.OMS.assign_resource_to_observatory_org(observatory_id, org_id) #find association org_objs = self.OMS.find_org_by_observatory(observatory_id) self.assertEqual(1, len(org_objs)) self.assertEqual(org_id, org_objs[0]._id) print("org_id=<" + org_id + ">") #create a subsite with parent Observatory subsite_obj = IonObject(RT.Subsite, name= 'TestSubsite', description = 'sample subsite') subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id) self.assertIsNotNone(subsite_id, "Subsite not created.") # verify that Subsite is linked to Observatory mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id) self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.") # add the Subsite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id) # verify that Subsite is linked to Org org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id) self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.") #create a logical platform with parent Subsite platform_site_obj = IonObject(RT.PlatformSite, name= 'TestPlatformSite', description = 'sample logical platform') platform_site_id = self.OMS.create_platform_site(platform_site_obj, subsite_id) self.assertIsNotNone(platform_site_id, "PlatformSite not created.") # verify that PlatformSite is linked to Site site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id) self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.") # add the PlatformSite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=platform_site_id, org_id=org_id) # verify that PlatformSite is linked to Org org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id) self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.") #create a logical instrument with parent logical platform instrument_site_obj = IonObject(RT.InstrumentSite, name= 'TestInstrumentSite', description = 'sample logical instrument') instrument_site_id = self.OMS.create_instrument_site(instrument_site_obj, platform_site_id) self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.") # verify that InstrumentSite is linked to PlatformSite li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id) self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.") # add the InstrumentSite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=instrument_site_id, org_id=org_id) # verify that InstrumentSite is linked to Org org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id) self.assertIsNotNone(org_li_assoc, "InstrumentSite not connected as resource to Org.") # remove the InstrumentSite as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(instrument_site_id, org_id) # verify that InstrumentSite is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True ) self.assertEqual(len(assocs), 0) # remove the InstrumentSite self.OMS.delete_instrument_site(instrument_site_id) assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True ) self.assertEqual(len(assocs), 1) #todo: remove the dangling association # remove the PlatformSite as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(platform_site_id, org_id) # verify that PlatformSite is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True ) self.assertEqual(len(assocs), 0) # remove the Site as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id) # verify that Site is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True ) self.assertEqual(len(assocs), 0) self.RR.delete(org_id) self.OMS.force_delete_observatory(observatory_id) self.OMS.force_delete_subsite(subsite_id) self.OMS.force_delete_platform_site(platform_site_id) self.OMS.force_delete_instrument_site(instrument_site_id) #@unittest.skip("in development...") @attr('EXT') def test_observatory_org_extended(self): stuff = self._make_associations() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product(input_resource_id=stuff.instrument_device_id, data_product_id=data_product_id1) #-------------------------------------------------------------------------------- # Get the extended Site (platformSite) #-------------------------------------------------------------------------------- extended_site = self.OMS.get_site_extension(stuff.platform_site_id) log.debug("extended_site: %s ", str(extended_site)) self.assertEqual(1, len(extended_site.platform_devices)) self.assertEqual(1, len(extended_site.platform_models)) self.assertEqual(stuff.platform_device_id, extended_site.platform_devices[0]._id) self.assertEqual(stuff.platform_model_id, extended_site.platform_models[0]._id) #-------------------------------------------------------------------------------- # Get the extended Org #-------------------------------------------------------------------------------- #test the extended resource extended_org = self.org_management_service.get_marine_facility_extension(stuff.org_id) log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org)) #self.assertEqual(2, len(extended_org.instruments_deployed) ) #self.assertEqual(1, len(extended_org.platforms_not_deployed) ) self.assertEqual(2, extended_org.number_of_platforms) self.assertEqual(2, len(extended_org.platform_models) ) self.assertEqual(2, extended_org.number_of_instruments) self.assertEqual(2, len(extended_org.instrument_models) ) #test the extended resource of the ION org ion_org_id = self.org_management_service.find_org() extended_org = self.org_management_service.get_marine_facility_extension(ion_org_id._id, user_id=12345) log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org)) self.assertEqual(0, len(extended_org.members)) self.assertEqual(0, extended_org.number_of_platforms) #self.assertEqual(1, len(extended_org.sites)) #-------------------------------------------------------------------------------- # Get the extended Site #-------------------------------------------------------------------------------- #create device state events to use for op /non-op filtering in extended t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device_id, state=ResourceAgentState.STREAMING ) self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE ) extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id) log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site)) self.dpclient.delete_data_product(data_product_id1)
class InstrumentSimulator(ApeComponent): def __init__(self, component_id, agent, configuration): ApeComponent.__init__(self, component_id, agent, configuration) model_id = None keep_running = True emit_granules = False thread = None def _start(self): # interact with container self.resource_registry = ResourceRegistryServiceClient(node=self.agent.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.agent.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.agent.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.agent.container.node) self.data_product_client = DataProductManagementServiceClient(node=self.agent.container.node) # add appropriate entries to the pycc registry if self.configuration.easy_registration: self.register() # begin loop to generate/send data self.thread = self._DataEmitter(self) self.thread.start() def _stop(self): log.debug("stopping instrument") self.stop_sending() self.keep_running = False def perform_action(self, request): log.debug('instrument simulator performing action ' + str(request)) if isinstance(request, StartRequest): self.start_sending() elif isinstance(request, StopRequest): self.stop_sending() elif isinstance(request, RegisterWithContainer): self.register() else: raise ApeException('instrument simulator does not know how to: ' + str(request)) def register(self): ''' create entries in resource registry for this instrument ''' # TODO: handle more than one ID returned from registry # TODO: fail if request register but already in registry (and not easy_registration) # TODO: should register device agent log.debug('registering instrument') product_ids,_ = self.resource_registry.find_resources(RT.DataProduct, None, self.configuration.data_product, id_only=True) if len(product_ids) > 0: log.debug('data product was already in the registry') self.data_product_id = product_ids[0] # if self.configuration.easy_registration: # self.data_product_id = product_ids[0] # else: # #raise ApeException('should not find data product in registry until i add it: ' + self.configuration.data_product) # ## TODO need ID from name: self.data_product_id = self.data_product_client.read_data_product(data_product=self.configuration.data_product) # pass else: log.debug('adding data product to the registry') # Create InstrumentDevice device = IonObject(RT.InstrumentDevice, name=self.configuration.data_product, description='instrument simulator', serial_number=self.configuration.data_product ) device_id = self.imsclient.create_instrument_device(instrument_device=device) self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id=self._get_model_id(), instrument_device_id=device_id) #@TODO: Do not use CoverageCraft ever. craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() data_product = IonObject(RT.DataProduct,name=self.configuration.data_product,description='ape producer', spatial_domain=sdom, temporal_domain=tdom) stream_def_id = self._get_streamdef_id(parameter_dictionary) self.data_product_id = self.data_product_client.create_data_product(data_product=data_product, stream_definition_id=stream_def_id) self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=self.data_product_id) if self.configuration.persist_product: self.data_product_client.activate_data_product_persistence(data_product_id=self.data_product_id, persist_data=True, persist_metadata=True) # get/create producer id # producer_ids, _ = self.clients.resource_registry.find_objects(self.data_product_id, PRED.hasDataProducer, RT.DataProducer, id_only=True) # if len(producer_ids)>0: # log.debug('data producer already in the registry') # self.data_producer_id = producer_ids[0] # else: self.data_producer_id = 'UNUSED' self._create_publisher() def _get_model_id(self): if self.model_id is None: try: self.model_id = self._read_model_id() self.model_id = self._read_model_id() except: model = IonObject(RT.InstrumentModel, name=self.configuration.instrument.model_name) #, description=self.configuration.model_name, model_label=self.data_source_name) self.model_id = self.imsclient.create_instrument_model(model) return self.model_id def _read_model_id(self): # debug reading instrument model try_this = self.resource_registry.find_resources(restype=RT.InstrumentModel, id_only=True) model_ids = self.resource_registry.find_resources(restype=RT.InstrumentModel, id_only=True, name=self.configuration.instrument.model_name) return model_ids[0][0] def _get_streamdef_id(self, pdict=None): #TODO: figure out how to read existing RAW definition instead of just creating #try: #stream_type = self.pubsubclient.read_stream_definition() stream_def_id = self.pubsubclient.create_stream_definition(name='RAW stream',parameter_dictionary=pdict) return stream_def_id def _create_publisher(self): stream_ids, _ = self.resource_registry.find_objects(self.data_product_id, PRED.hasStream, None, True) stream_id = stream_ids[0] self.publisher = StreamPublisher(process=self.agent, stream_id=stream_id) class _DataEmitter(Thread): ''' do not make outer class a Thread b/c start() is already meaningful to a pyon process ''' def __init__(self, instrument): Thread.__init__(self) self.instrument = instrument self.daemon = True def run(self): self.instrument.run() def start_sending(self): self.emit_granules = True def stop_sending(self): self.emit_granules = False def _get_parameter_dictionary(self): pdict = ParameterDictionary() cond_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=np.float64)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float64)) pres_ctxt.uom = 'unknown' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float64)) temp_ctxt.uom = 'unknown' temp_ctxt.fill_value = 0x0 pdict.add_context(temp_ctxt) oxy_ctxt = ParameterContext('oxygen', param_type=QuantityType(value_encoding=np.float64)) oxy_ctxt.uom = 'unknown' oxy_ctxt.fill_value = 0x0 pdict.add_context(oxy_ctxt) internal_ts_ctxt = ParameterContext(name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64)) internal_ts_ctxt._derived_from_name = 'time' internal_ts_ctxt.uom = 'seconds' internal_ts_ctxt.fill_value = -1 pdict.add_context(internal_ts_ctxt, is_temporal=True) driver_ts_ctxt = ParameterContext(name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64)) driver_ts_ctxt._derived_from_name = 'time' driver_ts_ctxt.uom = 'seconds' driver_ts_ctxt.fill_value = -1 pdict.add_context(driver_ts_ctxt) return pdict def run(self): try: self.do_run() except: log.exception('instrument simulator thread shutting down') def do_run(self): ''' main loop: essentially create granule, publish granule, then pause to remain at target rate. three lines of the useful code marked with ###, everything else is timing/reporting. ''' target_iteration_time = sleep_time = self.configuration.interval first_batch = True granule_count = iteration_count = granule_sum_size = 0 granule_elapsed_seconds = publish_elapsed_seconds = sleep_elapsed_seconds = iteration_elapsed_seconds = 0. do_timing = self.configuration.report_timing or self.configuration.log_timing parameter_dictionary = self._get_parameter_dictionary() #DatasetManagementServiceClient(node=self.agent.container.node).read_parameter_dictionary_by_name('ctd_parsed_param_dict') if not parameter_dictionary: log.error('failed to find parameter dictionary: ctd_parsed_param_dict (producer thread aborting)') return if type(parameter_dictionary) == dict: log.debug('pdict is dict') elif isinstance(parameter_dictionary,ParameterDictionary): log.debug('pdict is ParameterDictionary') else: log.warn('invalid pdict: %r', parameter_dictionary) parameter_dictionary = parameter_dictionary.__dict__ while self.keep_running: if self.emit_granules: adjust_timing = True iteration_start = granule_start = time() if do_timing: granule_count+=1 ### step 1: create granule granule = self.configuration.instrument.get_granule(time=time(), pd=parameter_dictionary) if do_timing: granule_end = publish_start = time() try: ### step 2: publish granule log.debug(self.component_id + ' publishing granule') self.publisher.publish(granule) except Exception as e: if self.keep_running: raise e else: # while blocking on publish(), async call may have closed connection # so eat exception and return cleanly return if do_timing: publish_end = time() granule_elapsed_seconds += (granule_end - granule_start) publish_elapsed_seconds += (publish_end - publish_start) pickled_granule = pickle.dumps(granule) granule_size = len(pickled_granule) granule_sum_size += granule_size sleep_start = time() ### step 3: sleep to maintain configured rate if sleep_time>0 or self.configuration.sleep_even_zero: sleep(sleep_time) if do_timing: sleep_end = time() sleep_elapsed_seconds += (sleep_end-sleep_start) if granule_count%self.configuration.timing_rate==0: # skip initial batch of timing -- can be skewed b/c one instrument start before the other, # so first instrument works against a more idle system if first_batch: granule_count = granule_elapsed_seconds = publish_elapsed_seconds = sleep_elapsed_seconds = 0. first_batch = False run_start_time = time() else: adjust_timing = False if self.configuration.log_timing: log.info('%s: granule: %f, publish: %f, sleep: %f, size %f' % (self.configuration.data_product, granule_elapsed_seconds/granule_count, publish_elapsed_seconds/granule_count, sleep_elapsed_seconds/granule_count, granule_sum_size/granule_count)) if self.configuration.report_timing: report = { 'count': granule_count, 'time': time() - run_start_time, 'publish':publish_elapsed_seconds/granule_count, 'granule':granule_elapsed_seconds/granule_count, 'sleep': sleep_elapsed_seconds/granule_count, 'iteration': iteration_elapsed_seconds/iteration_count } if self.configuration.include_size: report['size'] = granule_sum_size message = PerformanceResult(report) self.report(message) if adjust_timing: iteration_count+=1 iteration_end = time() actual_iteration_time = iteration_end - iteration_start iteration_elapsed_seconds += actual_iteration_time adjustment = target_iteration_time - actual_iteration_time sleep_time = max(sleep_time + adjustment, 0) else: # if not emitting granules, don't do timing either # and don't allow to sleep for too short a time and hog CPU sleep(max(10,self.configuration.interval)) log.debug('not emitting')
class TestPlatformLaunch(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient( node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient() self.IDS = IdentityManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # Use the network definition provided by RSN OMS directly. rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri']) self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms) # get serialized version for the configuration: self._network_definition_ser = NetworkUtil.serialize_network_definition( self._network_definition) if log.isEnabledFor(logging.TRACE): log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser) self._async_data_result = AsyncResult() self._data_subscribers = [] self._samples_received = [] self.addCleanup(self._stop_data_subscribers) self._async_event_result = AsyncResult() self._event_subscribers = [] self._events_received = [] self.addCleanup(self._stop_event_subscribers) self._start_event_subscriber() def _start_data_subscriber(self, stream_name, stream_id): """ Starts data subscriber for the given stream_name and stream_config """ def consume_data(message, stream_route, stream_id): # A callback for processing subscribed-to data. log.info('Subscriber received data message: %s.', str(message)) self._samples_received.append(message) self._async_data_result.set() log.info('_start_data_subscriber stream_name=%r stream_id=%r', stream_name, stream_id) # Create subscription for the stream exchange_name = '%s_queue' % stream_name self.container.ex_manager.create_xn_queue(exchange_name).purge() sub = StandaloneStreamSubscriber(exchange_name, consume_data) sub.start() self._data_subscribers.append(sub) sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id]) self.PSC.activate_subscription(sub_id) sub.subscription_id = sub_id def _stop_data_subscribers(self): """ Stop the data subscribers on cleanup. """ try: for sub in self._data_subscribers: if hasattr(sub, 'subscription_id'): try: self.PSC.deactivate_subscription(sub.subscription_id) except: pass self.PSC.delete_subscription(sub.subscription_id) sub.stop() finally: self._data_subscribers = [] def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"): """ Starts event subscriber for events of given event_type ("DeviceEvent" by default) and given sub_type ("platform_event" by default). """ def consume_event(evt, *args, **kwargs): # A callback for consuming events. log.info('Event subscriber received evt: %s.', str(evt)) self._events_received.append(evt) self._async_event_result.set(evt) sub = EventSubscriber(event_type=event_type, sub_type=sub_type, callback=consume_event) sub.start() log.info("registered event subscriber for event_type=%r, sub_type=%r", event_type, sub_type) self._event_subscribers.append(sub) sub._ready_event.wait(timeout=EVENT_TIMEOUT) def _stop_event_subscribers(self): """ Stops the event subscribers on cleanup. """ try: for sub in self._event_subscribers: if hasattr(sub, 'subscription_id'): try: self.PSC.deactivate_subscription(sub.subscription_id) except: pass self.PSC.delete_subscription(sub.subscription_id) sub.stop() finally: self._event_subscribers = [] def _create_platform_configuration(self): """ Verify that agent configurations are being built properly """ # # This method is an adaptation of test_agent_instance_config in # test_instrument_management_service_integration.py # clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC pconfig_builder = PlatformAgentConfigurationBuilder(clients) iconfig_builder = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_id = self.RR2.create(any_old(RT.Org)) inst_startup_config = {'startup': 'config'} required_config_keys = [ 'org_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'alarm_defs', 'children' ] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = { 'process_type': ('ZMQPyClassDriverLauncher', ), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher', )}, config['driver_config']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('stream_config', config) if None is inst_device_id: for key in ['alarm_defs', 'children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual('Org_1', config['org_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher', )}, config['driver_config']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('stream_config', config) for key in ['alarm_defs', 'startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name( 'platform_eng_parsed', id_only=True) self.parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_rpdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance( platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='platform_eng_parsed', records_per_granule=2, granule_publish_rate=5) platform_agent_obj = any_old( RT.PlatformAgent, {"stream_configurations": [raw_config]}) platform_agent_id = self.IMS.create_platform_agent( platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device( any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, { "temporal_domain": tdom, "spatial_domain": sdom }) # dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) dp_id = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device( platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance( platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource( platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old( RT.InstrumentAgentInstance, {"startup_config": inst_startup_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance( instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5) instrument_agent_obj = any_old( RT.InstrumentAgent, {"stream_configurations": [raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent( instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device( any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, { "temporal_domain": tdom, "spatial_domain": sdom }) dp_id = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product( input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device( instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance( instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource( instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug( "Testing that preparing a launcher without agent instance raises an error" ) self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False) log.debug( "Making the structure for a platform agent, which will be the child" ) platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure( ) platform_agent_instance_child_obj = self.RR2.read( platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") pconfig_builder.set_agent_instance_object( platform_agent_instance_child_obj) child_config = pconfig_builder.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug( "Making the structure for a platform agent, which will be the parent" ) platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure( ) platform_agent_instance_parent_obj = self.RR2.read( platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") pconfig_builder.set_agent_instance_object( platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") self.RR2.assign_platform_device_to_platform_device( platform_device_child_id, platform_device_parent_id) child_device_ids = self.RR2.find_platform_device_ids_of_device( platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") pconfig_builder.set_agent_instance_object( platform_agent_instance_parent_obj) parent_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure( ) instrument_agent_instance_obj = self.RR2.read( instrument_agent_instance_id) log.debug("Testing instrument config") iconfig_builder.set_agent_instance_object( instrument_agent_instance_obj) instrument_config = iconfig_builder.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device( instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device( platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") pconfig_builder.set_agent_instance_object( platform_agent_instance_parent_obj) full_config = pconfig_builder.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) if log.isEnabledFor(logging.TRACE): import pprint pp = pprint.PrettyPrinter() pp.pprint(full_config) log.trace("full_config = %s", pp.pformat(full_config)) return full_config def get_streamConfigs(self): # # This method is an adaptation of get_streamConfigs in # test_driver_egg.py # return [ StreamConfiguration( stream_name='parsed', parameter_dictionary_name='platform_eng_parsed', records_per_granule=2, granule_publish_rate=5) # TODO enable something like the following when also # incorporating "raw" data: #, #StreamConfiguration(stream_name='raw', # parameter_dictionary_name='ctd_raw_param_dict', # records_per_granule=2, # granule_publish_rate=5) ] @skip("Still needs alignment with new configuration structure") def test_hierarchy(self): # TODO re-implement. pass def test_single_platform(self): full_config = self._create_platform_configuration() platform_id = 'LJ01D' stream_configurations = self.get_streamConfigs() agent__obj = IonObject(RT.PlatformAgent, name='%s_PlatformAgent' % platform_id, description='%s_PlatformAgent platform agent' % platform_id, stream_configurations=stream_configurations) agent_id = self.IMS.create_platform_agent(agent__obj) device__obj = IonObject( RT.PlatformDevice, name='%s_PlatformDevice' % platform_id, description='%s_PlatformDevice platform device' % platform_id, # ports=port_objs, # platform_monitor_attributes = monitor_attribute_objs ) self.device_id = self.IMS.create_platform_device(device__obj) ####################################### # data product (adapted from test_instrument_management_service_integration) tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='DataProduct test', processing_level_code='Parsed_Canonical', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id) log.debug('data_product_id1 = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=self.device_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence( data_product_id=data_product_id1) ####################################### ####################################### # dataset stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] ####################################### full_config['platform_config'] = { 'platform_id': platform_id, 'driver_config': DVR_CONFIG, 'network_definition': self._network_definition_ser } agent_instance_obj = IonObject( RT.PlatformAgentInstance, name='%s_PlatformAgentInstance' % platform_id, description="%s_PlatformAgentInstance" % platform_id, agent_config=full_config) agent_instance_id = self.IMS.create_platform_agent_instance( platform_agent_instance=agent_instance_obj, platform_agent_id=agent_id, platform_device_id=self.device_id) stream_id = stream_ids[0] self._start_data_subscriber(agent_instance_id, stream_id) log.debug( "about to call imsclient.start_platform_agent_instance with id=%s", agent_instance_id) pid = self.IMS.start_platform_agent_instance( platform_agent_instance_id=agent_instance_id) log.debug("start_platform_agent_instance returned pid=%s", pid) #wait for start instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (90), "The platform agent instance did not spawn in 90 seconds") agent_instance_obj = self.IMS.read_instrument_agent_instance( agent_instance_id) log.debug('Platform agent instance obj') # Start a resource agent client to talk with the instrument agent. self._pa_client = ResourceAgentClient( 'paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("got platform agent client %s", str(self._pa_client)) # ping_agent can be issued before INITIALIZE retval = self._pa_client.ping_agent(timeout=TIMEOUT) log.debug('Base Platform ping_agent = %s', str(retval)) cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform INITIALIZE = %s', str(retval)) # GO_ACTIVE cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform GO_ACTIVE = %s', str(retval)) # RUN: cmd = AgentCommand(command=PlatformAgentEvent.RUN) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform RUN = %s', str(retval)) # START_MONITORING: cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform START_MONITORING = %s', str(retval)) # wait for data sample # just wait for at least one -- see consume_data above log.info("waiting for reception of a data sample...") self._async_data_result.get(timeout=DATA_TIMEOUT) self.assertTrue(len(self._samples_received) >= 1) log.info("waiting a bit more for reception of more data samples...") sleep(15) log.info("Got data samples: %d", len(self._samples_received)) # wait for event # just wait for at least one event -- see consume_event above log.info("waiting for reception of an event...") self._async_event_result.get(timeout=EVENT_TIMEOUT) log.info("Received events: %s", len(self._events_received)) #get the extended platfrom which wil include platform aggreate status fields # extended_platform = self.IMS.get_platform_device_extension(self.device_id) # log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) ) # log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) ) # log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) ) # STOP_MONITORING: cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform STOP_MONITORING = %s', str(retval)) # GO_INACTIVE cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform GO_INACTIVE = %s', str(retval)) # RESET: Resets the base platform agent, which includes termination of # its sub-platforms processes: cmd = AgentCommand(command=PlatformAgentEvent.RESET) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform RESET = %s', str(retval)) #------------------------------- # Stop Base Platform AgentInstance #------------------------------- self.IMS.stop_platform_agent_instance( platform_agent_instance_id=agent_instance_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dmpsclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.psmsclient = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) ) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.RR2.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_prepare_deployment_support(self): deploy_sup = self.omsclient.prepare_deployment_support() self.assertTrue(deploy_sup) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].type_, "AssocDeploymentInstDevice") self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].type_, "AssocDeploymentPlatDevice") self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].type_, "AssocDeploymentInstSite") self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].type_, "AssocDeploymentPlatSite") self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, []) #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1) self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, []) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources), 1) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, []) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1) self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources), 1) #delete the deployment self.RR2.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self): #------------------------------------------------------------------------------------- # Create platform site, platform device, platform model #------------------------------------------------------------------------------------- platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') platform_site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device(platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model(platform_model__obj) #------------------------------------------------------------------------------------- # Create instrument site #------------------------------------------------------------------------------------- instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) #---------------------------------------------------------------------------------------------------- # Create an instrument device #---------------------------------------------------------------------------------------------------- instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) #---------------------------------------------------------------------------------------------------- # Create an instrument model #---------------------------------------------------------------------------------------------------- instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj) #---------------------------------------------------------------------------------------------------- # Create a deployment object #---------------------------------------------------------------------------------------------------- start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_platform_model_to_platform_device(res.platform_model_id, res.platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_platform_model_to_platform_site(res.platform_model_id, res.platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("adding instrument site and device to deployment") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("adding platform site and device to deployment") self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id) self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) log.debug("deactivatin deployment, expecting success") self.omsclient.deactivate_deployment(res.deployment_id) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without site+device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without instrument site, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "Devices in this deployment outnumber sites") #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) log.debug("activating deployment without device, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment") def test_activate_deployment_asymmetric_children(self): """ P0 | \ P1 P2 | I1 Complex deployment using CSP P1, P2, and P3 share the same platform model. The CSP solver should be able to work this out based on relationships to parents """ log.debug("create models") imodel_id = self.RR2.create(any_old(RT.InstrumentModel)) pmodel_id = self.RR2.create(any_old(RT.PlatformModel)) log.debug("create devices") idevice_id = self.RR2.create(any_old(RT.InstrumentDevice)) pdevice_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(3)] log.debug("create sites") isite_id = self.RR2.create(any_old(RT.InstrumentSite)) psite_id = [self.RR2.create(any_old(RT.PlatformSite)) for _ in range(3)] log.debug("assign models") self.RR2.assign_instrument_model_to_instrument_device_with_has_model(imodel_id, idevice_id) self.RR2.assign_instrument_model_to_instrument_site_with_has_model(imodel_id, isite_id) for x in range(3): self.RR2.assign_platform_model_to_platform_device_with_has_model(pmodel_id, pdevice_id[x]) self.RR2.assign_platform_model_to_platform_site_with_has_model(pmodel_id, psite_id[x]) log.debug("assign hierarchy") self.RR2.assign_instrument_device_to_platform_device_with_has_device(idevice_id, pdevice_id[1]) self.RR2.assign_instrument_site_to_platform_site_with_has_site(isite_id, psite_id[1]) for x in range(1,3): self.RR2.assign_platform_device_to_platform_device_with_has_device(pdevice_id[x], pdevice_id[0]) self.RR2.assign_platform_site_to_platform_site_with_has_site(psite_id[x], psite_id[0]) log.debug("create and activate deployment") dep_id = self.RR2.create(any_old(RT.Deployment, {"context": IonObject(OT.RemotePlatformDeploymentContext)})) self.RR2.assign_deployment_to_platform_device_with_has_deployment(dep_id, pdevice_id[0]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(dep_id, psite_id[0]) self.omsclient.activate_deployment(dep_id) log.debug("verifying deployment") self.assertEqual(idevice_id, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(isite_id), "The instrument device was not assigned to the instrument site") for x in range(3): self.assertEqual(pdevice_id[x], self.RR2.find_platform_device_id_of_platform_site_using_has_device(psite_id[x]), "Platform device %d was not assigned to platform site %d" % (x, x)) def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message="did not specify fail_message"): with self.assertRaises(err_type) as cm: self.omsclient.activate_deployment(deployment_id) self.assertIn(fail_message, cm.exception.message) def test_3x3_matchups_remoteplatform(self): self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext)) def test_3x3_matchups_cabledinstrument(self): self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext)) def test_3x3_matchups_cablednode(self): self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext)) def base_3x3_matchups(self, deployment_context): """ This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1) """ deployment_context_type = type(deployment_context).__name__ instrument_model_id = [self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)] platform_model_id = [self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)] instrument_device_id = [self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)] platform_device_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)] instrument_site_id = [self.RR2.create(any_old(RT.InstrumentSite, {"planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="instport_%d" % (i+1))})) for i in range(9)] platform_site_id = [self.RR2.create(any_old(RT.PlatformSite, {"planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="platport_%d" % (i+1))})) for i in range(4)] def instrument_model_at(platform_idx, instrument_idx): m = platform_idx * 2 if instrument_idx > 0: m += 1 return m def platform_model_at(platform_idx): if platform_idx > 0: return 1 return 0 def instrument_at(platform_idx, instrument_idx): return platform_idx * 3 + instrument_idx # set up the structure for p in range(3): m = platform_model_at(p) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[m], platform_site_id[p]) self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[m], platform_device_id[p]) self.RR2.assign_platform_device_to_platform_device_with_has_device(platform_device_id[p], platform_device_id[3]) self.RR2.assign_platform_site_to_platform_site_with_has_site(platform_site_id[p], platform_site_id[3]) for i in range(3): m = instrument_model_at(p, i) idx = instrument_at(p, i) self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id[m], instrument_site_id[idx]) self.RR2.assign_instrument_model_to_instrument_device_with_has_model(instrument_model_id[m], instrument_device_id[idx]) self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id[idx], platform_device_id[p]) self.RR2.assign_instrument_site_to_platform_site_with_has_site(instrument_site_id[idx], platform_site_id[p]) # top level models self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[2], platform_device_id[3]) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[2], platform_site_id[3]) # verify structure for p in range(3): parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(platform_device_id[p]) self.assertEqual(platform_device_id[3], parent_id) parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(platform_site_id[p]) self.assertEqual(platform_site_id[3], parent_id) for i in range(len(platform_site_id)): self.assertEqual(self.RR2.find_platform_model_of_platform_device_using_has_model(platform_device_id[i]), self.RR2.find_platform_model_of_platform_site_using_has_model(platform_site_id[i])) for i in range(len(instrument_site_id)): self.assertEqual(self.RR2.find_instrument_model_of_instrument_device_using_has_model(instrument_device_id[i]), self.RR2.find_instrument_model_of_instrument_site_using_has_model(instrument_site_id[i])) port_assignments = {} for p in range(3): port_assignments[platform_device_id[p]] = "platport_%d" % (p+1) for i in range(3): idx = instrument_at(p, i) port_assignments[instrument_device_id[idx]] = "instport_%d" % (idx+1) deployment_id = self.RR2.create(any_old(RT.Deployment, {"context": deployment_context, "port_assignments": port_assignments})) log.debug("assigning device/site to %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[3]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[3]) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_instrument_device_with_has_deployment(deployment_id, instrument_device_id[1]) self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id[1]) elif OT.CabledNodeDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[1]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[1]) log.debug("activation of %s deployment", deployment_context_type) self.omsclient.activate_deployment(deployment_id) log.debug("validation of %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(d, self.RR2.find_platform_device_id_of_platform_site_using_has_device(platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual(d, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[i])) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.assertEqual(instrument_device_id[1], self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[1])) elif OT.CabledNodeDeploymentContext == deployment_context_type: expected_platforms = [1] expected_instruments = [3, 4, 5] # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(i in expected_platforms, d in self.RR2.find_platform_device_ids_of_platform_site_using_has_device(platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual(i in expected_instruments, d in self.RR2.find_instrument_device_ids_of_instrument_site_using_has_device(instrument_site_id[i]))
class TestAgentLaunchOps(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient( node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return def test_get_agent_client_noprocess(self): inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice)) iap = ResourceAgentClient._get_agent_process_id(inst_device_id) # should be no running agent self.assertIsNone(iap) # should raise NotFound self.assertRaises(NotFound, ResourceAgentClient, inst_device_id) def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.IMS.create_instrument_device( instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) spdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence( data_product_id=data_product_id1) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') data_product_id2 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence( data_product_id=data_product_id2) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance( instAgentInstance_id) gate = AgentProcessStateGate(self.PDC.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) if "BAD_DATA" == instance_obj.driver_config["comms_config"]: print "Saved config:" print snap_obj.content self.fail("Saved config was not properly restored") self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config_hasDevice(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(parent_device_id, PRED.hasDevice, child_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_objects(subject=parent_device_id, predicate=PRED.hasDevice, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasDevice") def test_agent_instance_config_hasNetworkParent(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(child_device_id, PRED.hasNetworkParent, parent_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_subjects(object=parent_device_id, predicate=PRED.hasNetworkParent, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasNetworkParent") def base_agent_instance_config( self, assign_child_platform_to_parent_platform_fn, find_child_platform_ids_of_parent_platform_fn): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC config_builder = DotDict config_builder.i = None config_builder.p = None def refresh_pconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.p = PlatformAgentConfigurationBuilder(clients) def refresh_iconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.i = InstrumentAgentConfigurationBuilder(clients) org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = [{'lvl2': 'lvl3val'}] required_config_keys = [ 'org_governance_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alerts_config', 'children' ] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = { 'process_type': ('ZMQPyClassDriverLauncher', ), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertIn('resource_id', config['agent']) self.assertEqual(device_id, config['agent']['resource_id']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) # TODO(OOIION-1495) review the asserts below related with # requiring 'ports' to be present in the driver_config. # See recent adjustment in agent_configuration_builder.py, # which I did to avoid other tests to fail. # The asserts below would make the following tests fail: # test_agent_instance_config_hasDevice # test_agent_instance_config_hasNetworkParent def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('resource_id', config['agent']) self.assertEqual(device_id, config['agent']['resource_id']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) """ self.assertIn('ports', config['driver_config']) """ self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher', ), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) """ if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) """ def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) """ self.assertIn('ports', config['driver_config']) """ self.assertEqual(('ZMQPyClassDriverLauncher', ), config['driver_config']['process_type']) self.assertIn('resource_id', config['agent']) self.assertEqual(parent_device_id, config['agent']['resource_id']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) """ if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) """ self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(name='', agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old( RT.PlatformAgentInstance, { 'driver_config': { 'foo': 'bar' }, 'alerts': generic_alerts_config }) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance( platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') platform_agent_obj = any_old( RT.PlatformAgent, {"stream_configurations": [raw_config]}) platform_agent_id = self.IMS.create_platform_agent( platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device( any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct) dp_id = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.PlatformSite, name='sitePlatform') site_id = self.OMS.create_platform_site(platform_site=site_obj) # find current deployment using time constraints current_time = int(calendar.timegm(time.gmtime())) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj = IonObject( OT.PlatformPort, reference_designator='GA01SUMO-FI003-09-CTDMO0999', port_type=PortTypeEnum.UPLINK, ip_address=0) deployment_obj = IonObject( RT.Deployment, name='TestPlatformDeployment_' + name, description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds], port_assignments={platform_device_id: platform_port_obj}) deploy_id = self.OMS.create_deployment( deployment=deployment_obj, site_id=site_id, device_id=platform_device_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance( platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition( platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource( platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old( RT.InstrumentAgentInstance, { "startup_config": inst_startup_config, 'alerts': generic_alerts_config }) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance( instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') instrument_agent_obj = any_old( RT.InstrumentAgent, {"stream_configurations": [raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent( instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device( any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct) dp_id = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product( input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.InstrumentSite, name='siteInstrument') site_id = self.OMS.create_instrument_site(instrument_site=site_obj) # find current deployment using time constraints current_time = int(calendar.timegm(time.gmtime())) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj = IonObject( OT.PlatformPort, reference_designator='GA01SUMO-FI003-08-CTDMO0888', port_type=PortTypeEnum.PAYLOAD, ip_address=0) deployment_obj = IonObject( RT.Deployment, name='TestDeployment for Cabled Instrument', description='some new deployment', context=IonObject(OT.CabledInstrumentDeploymentContext), constraint_list=[temporal_bounds], port_assignments={instrument_device_id: platform_port_obj}) deploy_id = self.OMS.create_deployment( deployment=deployment_obj, site_id=site_id, device_id=instrument_device_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance( instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition( instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource( instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug( "Testing that preparing a launcher without agent instance raises an error" ) refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated self.assertRaises(AssertionError, config_builder.p.prepare, will_launch=False) log.debug( "Making the structure for a platform agent, which will be the child" ) platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure( name='child') platform_agent_instance_child_obj = self.RR2.read( platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_child_obj) child_config = config_builder.p.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug( "Making the structure for a platform agent, which will be the parent" ) platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure( name='parent') platform_agent_instance_parent_obj = self.RR2.read( platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") assign_child_platform_to_parent_platform_fn(platform_device_child_id, platform_device_parent_id) child_device_ids = find_child_platform_ids_of_parent_platform_fn( platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure( ) instrument_agent_instance_obj = self.RR2.read( instrument_agent_instance_id) log.debug("Testing instrument config") refresh_iconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.i.set_agent_instance_object( instrument_agent_instance_obj) instrument_config = config_builder.i.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device_with_has_device( instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_platform_device_using_has_device( platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") refresh_pconfig_builder_hack( config_builder ) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object( platform_agent_instance_parent_obj) full_config = config_builder.p.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) log.info("END base_agent_instance_config")
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.org_management_service = OrgManagementServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #print 'TestObservatoryManagementServiceIntegration: started services' self.event_publisher = EventPublisher() # @unittest.skip('this exists only for debugging the launch process') # def test_just_the_setup(self): # return def destroy(self, resource_ids): self.OMS.force_delete_observatory(resource_ids.observatory_id) self.OMS.force_delete_subsite(resource_ids.subsite_id) self.OMS.force_delete_subsite(resource_ids.subsite2_id) self.OMS.force_delete_subsite(resource_ids.subsiteb_id) self.OMS.force_delete_subsite(resource_ids.subsitez_id) self.OMS.force_delete_platform_site(resource_ids.platform_site_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id) self.OMS.force_delete_platform_site(resource_ids.platform_site3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_siteb3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id) #@unittest.skip('targeting') def test_observatory_management(self): resources = self._make_associations() self._do_test_find_related_sites(resources) self._do_test_get_sites_devices_status(resources) self._do_test_find_site_data_products(resources) self._do_test_find_related_frames_of_reference(resources) self._do_test_create_geospatial_point_center(resources) self._do_test_find_observatory_org(resources) self.destroy(resources) def _do_test_find_related_sites(self, resources): site_resources, site_children = self.OMS.find_related_sites(resources.org_id) #import sys, pprint #print >> sys.stderr, pprint.pformat(site_resources) #print >> sys.stderr, pprint.pformat(site_children) #self.assertIn(resources.org_id, site_resources) self.assertIn(resources.observatory_id, site_resources) self.assertIn(resources.subsite_id, site_resources) self.assertIn(resources.subsite_id, site_resources) self.assertIn(resources.subsite2_id, site_resources) self.assertIn(resources.platform_site_id, site_resources) self.assertIn(resources.instrument_site_id, site_resources) self.assertEquals(len(site_resources), 13) self.assertEquals(site_resources[resources.observatory_id].type_, RT.Observatory) self.assertIn(resources.org_id, site_children) self.assertIn(resources.observatory_id, site_children) self.assertIn(resources.subsite_id, site_children) self.assertIn(resources.subsite_id, site_children) self.assertIn(resources.subsite2_id, site_children) self.assertIn(resources.platform_site_id, site_children) self.assertNotIn(resources.instrument_site_id, site_children) self.assertEquals(len(site_children), 9) self.assertIsInstance(site_children[resources.subsite_id], list) self.assertEquals(len(site_children[resources.subsite_id]), 2) def _do_test_get_sites_devices_status(self, resources): result_dict = self.OMS.get_sites_devices_status(resources.org_id) site_resources = result_dict.get("site_resources", None) site_children = result_dict.get("site_children", None) self.assertEquals(len(site_resources), 14) self.assertEquals(len(site_children), 9) result_dict = self.OMS.get_sites_devices_status(resources.org_id, include_devices=True, include_status=True) log.debug("RESULT DICT: %s", result_dict.keys()) site_resources = result_dict.get("site_resources", None) site_children = result_dict.get("site_children", None) site_status = result_dict.get("site_status", None) self.assertEquals(len(site_resources), 14) self.assertEquals(len(site_children), 9) result_dict = self.OMS.get_sites_devices_status(resources.observatory_id, include_devices=True, include_status=True) site_resources = result_dict.get("site_resources") site_children = result_dict.get("site_children") site_status = result_dict.get("site_status") self.assertEquals(len(site_resources), 13) self.assertEquals(len(site_children), 8) def _do_test_find_site_data_products(self, resources): res_dict = self.OMS.find_site_data_products(resources.org_id) #import sys, pprint #print >> sys.stderr, pprint.pformat(res_dict) self.assertIsNone(res_dict['data_product_resources']) self.assertIn(resources.platform_device_id, res_dict['device_data_products']) self.assertIn(resources.instrument_device_id, res_dict['device_data_products']) #@unittest.skip('targeting') def _do_test_find_related_frames_of_reference(self, stuff): # finding subordinates gives a dict of obj lists, convert objs to ids def idify(adict): ids = {} for k, v in adict.iteritems(): ids[k] = [] for obj in v: ids[k].append(obj._id) return ids # a short version of the function we're testing, with id-ify def short(resource_id, output_types): ret = self.OMS.find_related_frames_of_reference(resource_id, output_types) return idify(ret) #set up associations first stuff = self._make_associations() #basic traversal of tree from instrument to platform ids = short(stuff.instrument_site_id, [RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) #since this is the first search, just make sure the input inst_id got stripped if RT.InstrumentSite in ids: self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #basic traversal of tree from platform to instrument ids = short(stuff.platform_siteb_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite]) #full traversal of tree from observatory down to instrument ids = short(stuff.observatory_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #full traversal of tree from instrument to observatory ids = short(stuff.instrument_site_id, [RT.Observatory]) self.assertIn(RT.Observatory, ids) self.assertIn(stuff.observatory_id, ids[RT.Observatory]) #partial traversal, only down to platform ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(RT.InstrumentSite, ids) #partial traversal, only down to platform ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertNotIn(RT.Observatory, ids) self.destroy(stuff) def _make_associations(self): """ create one of each resource and association used by OMS to guard against problems in ion-definitions """ #raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41") """ the tree we're creating (observatory, sites, platforms, instruments) rows are lettered, colums numbered. - first row is implied a - first column is implied 1 - site Z, just because O--Sz | S--S2--P3--I4 | Sb-Pb2-Ib3 | P--I2 <- PlatformDevice, InstrumentDevice2 | Pb <- PlatformDevice b | I <- InstrumentDevice """ org_id = self.OMS.create_marine_facility(any_old(RT.Org)) def create_under_org(resource_type, extra_fields=None): obj = any_old(resource_type, extra_fields) if RT.InstrumentDevice == resource_type: resource_id = self.IMS.create_instrument_device(obj) else: resource_id, _ = self.RR.create(obj) self.OMS.assign_resource_to_observatory_org(resource_id=resource_id, org_id=org_id) return resource_id #stuff we control observatory_id = create_under_org(RT.Observatory) subsite_id = create_under_org(RT.Subsite) subsite2_id = create_under_org(RT.Subsite) subsiteb_id = create_under_org(RT.Subsite) subsitez_id = create_under_org(RT.Subsite) platform_site_id = create_under_org(RT.PlatformSite) platform_siteb_id = create_under_org(RT.PlatformSite) platform_siteb2_id = create_under_org(RT.PlatformSite) platform_site3_id = create_under_org(RT.PlatformSite) instrument_site_id = create_under_org(RT.InstrumentSite) instrument_site2_id = create_under_org(RT.InstrumentSite) instrument_siteb3_id = create_under_org(RT.InstrumentSite) instrument_site4_id = create_under_org(RT.InstrumentSite) #stuff we associate to instrument_device_id = create_under_org(RT.InstrumentDevice) instrument_device2_id = create_under_org(RT.InstrumentDevice) platform_device_id = create_under_org(RT.PlatformDevice) platform_deviceb_id = create_under_org(RT.PlatformDevice) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) deployment_id, _ = self.RR.create(any_old(RT.Deployment)) #observatory self.RR.create_association(observatory_id, PRED.hasSite, subsite_id) self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id) #site self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id) self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id) self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id) #platform_site(s) self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id) self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id) self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id) self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id) self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id) #test network parent link self.OMS.assign_device_to_network_parent(platform_device_id, platform_deviceb_id) self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id) #instrument_site(s) self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id) self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) ret = DotDict() ret.org_id = org_id ret.observatory_id = observatory_id ret.subsite_id = subsite_id ret.subsite2_id = subsite2_id ret.subsiteb_id = subsiteb_id ret.subsitez_id = subsitez_id ret.platform_site_id = platform_site_id ret.platform_siteb_id = platform_siteb_id ret.platform_siteb2_id = platform_siteb2_id ret.platform_site3_id = platform_site3_id ret.instrument_site_id = instrument_site_id ret.instrument_site2_id = instrument_site2_id ret.instrument_siteb3_id = instrument_siteb3_id ret.instrument_site4_id = instrument_site4_id ret.instrument_device_id = instrument_device_id ret.instrument_device2_id = instrument_device2_id ret.platform_device_id = platform_device_id ret.platform_deviceb_id = platform_deviceb_id ret.instrument_model_id = instrument_model_id ret.platform_model_id = platform_model_id ret.deployment_id = deployment_id return ret #@unittest.skip("targeting") def test_create_observatory(self): observatory_obj = IonObject(RT.Observatory, name='TestFacility', description='some new mf') observatory_id = self.OMS.create_observatory(observatory_obj) self.OMS.force_delete_observatory(observatory_id) #@unittest.skip("targeting") def _do_test_create_geospatial_point_center(self, resources): platformsite_obj = IonObject(RT.PlatformSite, name='TestPlatformSite', description='some new TestPlatformSite') geo_index_obj = IonObject(OT.GeospatialBounds) geo_index_obj.geospatial_latitude_limit_north = 20.0 geo_index_obj.geospatial_latitude_limit_south = 10.0 geo_index_obj.geospatial_longitude_limit_east = 15.0 geo_index_obj.geospatial_longitude_limit_west = 20.0 platformsite_obj.constraint_list = [geo_index_obj] platformsite_id = self.OMS.create_platform_site(platformsite_obj) # now get the dp back to see if it was updated platformsite_obj = self.OMS.read_platform_site(platformsite_id) self.assertEquals('some new TestPlatformSite', platformsite_obj.description) self.assertAlmostEqual(15.0, platformsite_obj.geospatial_point_center.lat, places=1) #now adjust a few params platformsite_obj.description = 'some old TestPlatformSite' geo_index_obj = IonObject(OT.GeospatialBounds) geo_index_obj.geospatial_latitude_limit_north = 30.0 geo_index_obj.geospatial_latitude_limit_south = 20.0 platformsite_obj.constraint_list = [geo_index_obj] update_result = self.OMS.update_platform_site(platformsite_obj) # now get the dp back to see if it was updated platformsite_obj = self.OMS.read_platform_site(platformsite_id) self.assertEquals('some old TestPlatformSite', platformsite_obj.description) self.assertAlmostEqual(25.0, platformsite_obj.geospatial_point_center.lat, places=1) self.OMS.force_delete_platform_site(platformsite_id) #@unittest.skip("targeting") def _do_test_find_observatory_org(self, resources): log.debug("Make TestOrg") org_obj = IonObject(RT.Org, name='TestOrg', description='some new mf org') org_id = self.OMS.create_marine_facility(org_obj) log.debug("Make Observatory") observatory_obj = IonObject(RT.Observatory, name='TestObservatory', description='some new obs') observatory_id = self.OMS.create_observatory(observatory_obj) log.debug("assign observatory to org") self.OMS.assign_resource_to_observatory_org(observatory_id, org_id) log.debug("verify assigment") org_objs = self.OMS.find_org_by_observatory(observatory_id) self.assertEqual(1, len(org_objs)) self.assertEqual(org_id, org_objs[0]._id) log.debug("org_id=<" + org_id + ">") log.debug("create a subsite with parent Observatory") subsite_obj = IonObject(RT.Subsite, name= 'TestSubsite', description = 'sample subsite') subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id) self.assertIsNotNone(subsite_id, "Subsite not created.") log.debug("verify that Subsite is linked to Observatory") mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id) self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.") log.debug("add the Subsite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id) log.debug("verify that Subsite is linked to Org") org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id) self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.") log.debug("create a logical platform with parent Subsite") platform_site_obj = IonObject(RT.PlatformSite, name= 'TestPlatformSite', description = 'sample logical platform') platform_site_id = self.OMS.create_platform_site(platform_site_obj, subsite_id) self.assertIsNotNone(platform_site_id, "PlatformSite not created.") log.debug("verify that PlatformSite is linked to Site") site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id) self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.") log.debug("add the PlatformSite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org(resource_id=platform_site_id, org_id=org_id) log.debug("verify that PlatformSite is linked to Org") org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id) self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.") log.debug("create a logical instrument with parent logical platform") instrument_site_obj = IonObject(RT.InstrumentSite, name= 'TestInstrumentSite', description = 'sample logical instrument') instrument_site_id = self.OMS.create_instrument_site(instrument_site_obj, platform_site_id) self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.") log.debug("verify that InstrumentSite is linked to PlatformSite") li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id) self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.") log.debug("add the InstrumentSite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org(resource_id=instrument_site_id, org_id=org_id) log.debug("verify that InstrumentSite is linked to Org") org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id) self.assertIsNotNone(org_li_assoc, "InstrumentSite not connected as resource to Org.") log.debug("remove the InstrumentSite as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org(instrument_site_id, org_id) log.debug("verify that InstrumentSite is linked to Org") assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True ) self.assertEqual(0, len(assocs)) log.debug("remove the InstrumentSite, association should drop automatically") self.OMS.delete_instrument_site(instrument_site_id) assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True ) self.assertEqual(0, len(assocs)) log.debug("remove the PlatformSite as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org(platform_site_id, org_id) log.debug("verify that PlatformSite is linked to Org") assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True ) self.assertEqual(0, len(assocs)) log.debug("remove the Site as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id) log.debug("verify that Site is linked to Org") assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True ) self.assertEqual(0, len(assocs)) self.RR.delete(org_id) self.OMS.force_delete_observatory(observatory_id) self.OMS.force_delete_subsite(subsite_id) self.OMS.force_delete_platform_site(platform_site_id) self.OMS.force_delete_instrument_site(instrument_site_id) @attr('EXT') def test_observatory_extensions(self): obs_id = self.RR2.create(any_old(RT.Observatory)) pss_id = self.RR2.create(any_old(RT.PlatformSite, dict(alt_resource_type="StationSite"))) pas_id = self.RR2.create(any_old(RT.PlatformSite, dict(alt_resource_type="PlatformAssemblySite"))) pcs_id = self.RR2.create(any_old(RT.PlatformSite, dict(alt_resource_type="PlatformComponentSite"))) ins_id = self.RR2.create(any_old(RT.InstrumentSite)) obs_obj = self.RR2.read(obs_id) pss_obj = self.RR2.read(pss_id) pas_obj = self.RR2.read(pas_id) pcs_obj = self.RR2.read(pcs_id) ins_obj = self.RR2.read(ins_id) self.RR2.create_association(obs_id, PRED.hasSite, pss_id) self.RR2.create_association(pss_id, PRED.hasSite, pas_id) self.RR2.create_association(pas_id, PRED.hasSite, pcs_id) self.RR2.create_association(pcs_id, PRED.hasSite, ins_id) extended_obs = self.OMS.get_observatory_site_extension(obs_id, user_id=12345) self.assertEqual([pss_obj], extended_obs.computed.platform_station_sites.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_obs.computed.platform_station_sites.status) self.assertEqual([pas_obj], extended_obs.computed.platform_assembly_sites.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_obs.computed.platform_assembly_sites.status) self.assertEqual([pcs_obj], extended_obs.computed.platform_component_sites.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_obs.computed.platform_component_sites.status) self.assertEqual([ins_obj], extended_obs.computed.instrument_sites.value) extended_pss = self.OMS.get_observatory_site_extension(obs_id, user_id=12345) self.assertEqual([pas_obj], extended_pss.computed.platform_assembly_sites.value) self.assertEqual([pcs_obj], extended_pss.computed.platform_component_sites.value) self.assertEqual([ins_obj], extended_pss.computed.instrument_sites.value) extended_pas = self.OMS.get_observatory_site_extension(pas_id, user_id=12345) self.assertEqual([pcs_obj], extended_pas.computed.platform_component_sites.value) self.assertEqual([ins_obj], extended_pas.computed.instrument_sites.value) extended_pcs = self.OMS.get_platform_component_site_extension(pcs_id, user_id=12345) self.assertEqual([ins_obj], extended_pcs.computed.instrument_sites.value) #@unittest.skip("in development...") @attr('EXT') @attr('EXT1') def test_observatory_org_extended(self): stuff = self._make_associations() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product(input_resource_id=stuff.instrument_device_id, data_product_id=data_product_id1) #Create a user to be used as regular member member_actor_obj = IonObject(RT.ActorIdentity, name='org member actor') member_actor_id,_ = self.RR.create(member_actor_obj) assert(member_actor_id) member_actor_header = get_actor_header(member_actor_id) member_user_obj = IonObject(RT.UserInfo, name='org member user') member_user_id,_ = self.RR.create(member_user_obj) assert(member_user_id) self.RR.create_association(subject=member_actor_id, predicate=PRED.hasInfo, object=member_user_id) #Build the Service Agreement Proposal to enroll a user actor sap = IonObject(OT.EnrollmentProposal,consumer=member_actor_id, provider=stuff.org_id ) sap_response = self.org_management_service.negotiate(sap, headers=member_actor_header ) #enroll the member without using negotiation self.org_management_service.enroll_member(org_id=stuff.org_id, actor_id=member_actor_id) #-------------------------------------------------------------------------------- # Get the extended Site (platformSite) #-------------------------------------------------------------------------------- try: extended_site = self.OMS.get_site_extension(stuff.platform_site_id) except: log.error('failed to get extended site', exc_info=True) raise log.debug("extended_site: %r ", extended_site) self.assertEqual(1, len(extended_site.platform_devices)) self.assertEqual(1, len(extended_site.platform_models)) self.assertEqual(stuff.platform_device_id, extended_site.platform_devices[0]._id) self.assertEqual(stuff.platform_model_id, extended_site.platform_models[0]._id) log.debug("verify that PlatformDeviceb is linked to PlatformDevice with hasNetworkParent link") associations = self.RR.find_associations(subject=stuff.platform_deviceb_id, predicate=PRED.hasNetworkParent, object=stuff.platform_device_id, id_only=True) self.assertIsNotNone(associations, "PlatformDevice child not connected to PlatformDevice parent.") #-------------------------------------------------------------------------------- # Get the extended Org #-------------------------------------------------------------------------------- #test the extended resource extended_org = self.OMS.get_marine_facility_extension(stuff.org_id) log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org)) #self.assertEqual(2, len(extended_org.instruments_deployed) ) #self.assertEqual(1, len(extended_org.platforms_not_deployed) ) self.assertEqual(2, extended_org.number_of_platforms) self.assertEqual(2, len(extended_org.platform_models) ) self.assertEqual(2, extended_org.number_of_instruments) self.assertEqual(2, len(extended_org.instrument_models) ) self.assertEqual(1, len(extended_org.members)) self.assertNotEqual(extended_org.members[0]._id, member_actor_id) self.assertEqual(extended_org.members[0]._id, member_user_id) self.assertEqual(1, len(extended_org.open_requests)) self.assertTrue(len(extended_site.deployments)>0) self.assertEqual(len(extended_site.deployments), len(extended_site.deployment_info)) #test the extended resource of the ION org ion_org_id = self.org_management_service.find_org() extended_org = self.OMS.get_marine_facility_extension(ion_org_id._id, user_id=12345) log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org)) self.assertEqual(1, len(extended_org.members)) self.assertEqual(0, extended_org.number_of_platforms) #self.assertEqual(1, len(extended_org.sites)) #-------------------------------------------------------------------------------- # Get the extended Site #-------------------------------------------------------------------------------- #create device state events to use for op /non-op filtering in extended t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device_id, state=ResourceAgentState.STREAMING ) self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE ) extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id) log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site)) self.dpclient.delete_data_product(data_product_id1)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment( deployment_obj, site_id, device_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj)) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.omsclient.delete_deployment(deployment_id) # now try to get the deleted dp object try: deployment_obj = self.omsclient.read_deployment(deployment_id) except NotFound as ex: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_activate_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') model_id = self.imsclient.create_platform_model(platform_model__obj) self.imsclient.assign_platform_model_to_platform_device( model_id, device_id) self.omsclient.assign_platform_model_to_platform_site( model_id, site_id) #create a deployment with metadata and an initial site and device instrument_site__obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site( instrument_site__obj, site_id) instrument_device__obj = IonObject( RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device__obj) self.rrclient.create_association(device_id, PRED.hasDevice, instrument_device_id) instrument_model__obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model( instrument_model__obj) self.imsclient.assign_instrument_model_to_instrument_device( instrument_model_id, instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( instrument_model_id, instrument_site_id) #self.rrclient.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment( deployment_obj, site_id, device_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) self.omsclient.activate_deployment(deployment_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dmpsclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.psmsclient = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.resource_impl = ResourceImpl(self.c) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) ) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.resource_impl.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: deployment_obj = self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_activate_deployment(self): #------------------------------------------------------------------------------------- # Create platform site, platform device, platform model #------------------------------------------------------------------------------------- platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device(platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') model_id = self.imsclient.create_platform_model(platform_model__obj) #------------------------------------------------------------------------------------- # Assign platform model to platform device and site #------------------------------------------------------------------------------------- self.imsclient.assign_platform_model_to_platform_device(model_id, platform_device_id) self.omsclient.assign_platform_model_to_platform_site(model_id, site_id) #------------------------------------------------------------------------------------- # Create instrument site #------------------------------------------------------------------------------------- instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='Log Data Product', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) out_log_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id) #---------------------------------------------------------------------------------------------------- # Start the transform (a logical transform) that acts as an instrument site #---------------------------------------------------------------------------------------------------- self.omsclient.create_site_data_product( site_id= instrument_site_id, data_product_id = out_log_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument device #---------------------------------------------------------------------------------------------------- instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) dp_obj = IonObject(RT.DataProduct, name='Instrument Data Product', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) inst_data_product_id = self.dmpsclient.create_data_product(dp_obj, ctd_stream_def_id) #assign data products appropriately self.damsclient.assign_data_product(input_resource_id=instrument_device_id, data_product_id=inst_data_product_id) #---------------------------------------------------------------------------------------------------- # Create an instrument model #---------------------------------------------------------------------------------------------------- instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj) self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(instrument_model_id, instrument_site_id) #---------------------------------------------------------------------------------------------------- # Create a deployment object #---------------------------------------------------------------------------------------------------- start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instrument_device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id)
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) #@unittest.skip('not working') def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', deviceAttachment #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- driver_config = { 'dvr_mod' : 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls' : 'SBE37Driver', 'workdir' : '/tmp/', } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) print 'TestDataProcessWithLookupTable: new Stream Definition id = ', instDevice_id print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test') try: ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) print 'TestDataProcessWithLookupTable: Data product streams1 = ', stream_ids #------------------------------- # Create CTD Raw as the second data product #------------------------------- print 'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition' raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def) dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test') try: ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new ctd_raw_data_product_id = ', ctd_raw_data_product self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) ) processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) ) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products={} log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity') ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True) log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure') ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True) log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature') ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, in_prods, self.output_products) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProcessWithLookupTable: create L0 all data_process return") contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) print 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = ', processAttachment
class TestAgentLaunchOps(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return def test_get_agent_client_noprocess(self): inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice)) iap = ResourceAgentClient._get_agent_process_id(inst_device_id) # should be no running agent self.assertIsNone(iap) # should raise NotFound self.assertRaises(NotFound, ResourceAgentClient, inst_device_id) def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = AgentProcessStateGate(self.PDC.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) if "BAD_DATA" == instance_obj.driver_config["comms_config"]: print "Saved config:" print snap_obj.content self.fail("Saved config was not properly restored") self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2) def test_agent_instance_config_hasDevice(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(parent_device_id, PRED.hasDevice, child_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_objects(subject=parent_device_id, predicate=PRED.hasDevice, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasDevice") def test_agent_instance_config_hasNetworkParent(self): def assign_fn(child_device_id, parent_device_id): self.RR2.create_association(child_device_id, PRED.hasNetworkParent, parent_device_id) def find_fn(parent_device_id): ret, _ = self.RR.find_subjects(object=parent_device_id, predicate=PRED.hasNetworkParent, id_only=True) return ret self.base_agent_instance_config(assign_fn, find_fn) log.info("END test_agent_instance_config_hasNetworkParent") def base_agent_instance_config(self, assign_child_platform_to_parent_platform_fn, find_child_platform_ids_of_parent_platform_fn): """ Verify that agent configurations are being built properly """ clients = DotDict() clients.resource_registry = self.RR clients.pubsub_management = self.PSC clients.dataset_management = self.DSC config_builder = DotDict config_builder.i = None config_builder.p = None def refresh_pconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.p = PlatformAgentConfigurationBuilder(clients) def refresh_iconfig_builder_hack(config_builder): """ ugly hack to get around "idempotent" RR2 caching remove after https://github.com/ooici/coi-services/pull/1190 """ config_builder.i = InstrumentAgentConfigurationBuilder(clients) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_obj = any_old(RT.Org) org_id = self.RR2.create(org_obj) inst_startup_config = {'startup': 'config'} generic_alerts_config = [ {'lvl2': 'lvl3val'} ] required_config_keys = [ 'org_governance_name', 'device_type', 'agent', 'driver_config', 'stream_config', 'startup_config', 'aparam_alerts_config', 'children'] def verify_instrument_config(config, device_id): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.InstrumentDevice, config['device_type']) self.assertIn('driver_config', config) driver_config = config['driver_config'] expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',), } for k, v in expected_driver_fields.iteritems(): self.assertIn(k, driver_config) self.assertEqual(v, driver_config[k]) self.assertEqual self.assertEqual({'resource_id': device_id}, config['agent']) self.assertEqual(inst_startup_config, config['startup_config']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['children']: self.assertEqual({}, config[key]) def verify_child_config(config, device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) self.assertIn('driver_config', config) self.assertIn('foo', config['driver_config']) self.assertIn('ports', config['driver_config']) self.assertEqual('bar', config['driver_config']['foo']) self.assertIn('process_type', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) if None is inst_device_id: for key in ['children', 'startup_config']: self.assertEqual({}, config[key]) else: for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(inst_device_id, config['children']) verify_instrument_config(config['children'][inst_device_id], inst_device_id) if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(org_obj.org_governance_name, config['org_governance_name']) self.assertEqual(RT.PlatformDevice, config['device_type']) self.assertIn('process_type', config['driver_config']) self.assertIn('ports', config['driver_config']) self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('aparam_alerts_config', config) self.assertEqual(generic_alerts_config, config['aparam_alerts_config']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) if config['driver_config']['ports']: self.assertTrue( isinstance(config['driver_config']['ports'], dict) ) self.assertIn(child_device_id, config['children']) verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #todo: create org and figure out which agent resource needs to get assigned to it def _make_platform_agent_structure(name='', agent_config=None): if None is agent_config: agent_config = {} # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'}, 'alerts': generic_alerts_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' ) platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.PlatformSite, name='sitePlatform') site_id = self.OMS.create_platform_site(platform_site=site_obj) # find current deployment using time constraints current_time = int( calendar.timegm(time.gmtime()) ) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-09-CTDMO0999', port_type=PortTypeEnum.UPLINK, ip_address=0) deployment_obj = IonObject(RT.Deployment, name='TestPlatformDeployment_' + name, description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds], port_assignments={platform_device_id:platform_port_obj}) deploy_id = self.OMS.create_deployment(deployment=deployment_obj, site_id=site_id, device_id=platform_device_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id) return platform_agent_instance_id, platform_agent_id, platform_device_id def _make_instrument_agent_structure(agent_config=None): if None is agent_config: agent_config = {} # instance creation instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config, 'alerts': generic_alerts_config}) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # agent creation raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' ) instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]}) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) # device creation instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.suspend_data_product_persistence, dp_id) #deployment creation site_obj = IonObject(RT.InstrumentSite, name='siteInstrument') site_id = self.OMS.create_instrument_site(instrument_site =site_obj) # find current deployment using time constraints current_time = int( calendar.timegm(time.gmtime()) ) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-08-CTDMO0888', port_type=PortTypeEnum.PAYLOAD, ip_address=0) deployment_obj = IonObject(RT.Deployment, name='TestDeployment for Cabled Instrument', description='some new deployment', context=IonObject(OT.CabledInstrumentDeploymentContext), constraint_list=[temporal_bounds], port_assignments={instrument_device_id:platform_port_obj}) deploy_id = self.OMS.create_deployment(deployment=deployment_obj, site_id=site_id, device_id=instrument_device_id) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) return instrument_agent_instance_id, instrument_agent_id, instrument_device_id # can't do anything without an agent instance obj log.debug("Testing that preparing a launcher without agent instance raises an error") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated self.assertRaises(AssertionError, config_builder.p.prepare, will_launch=False) log.debug("Making the structure for a platform agent, which will be the child") platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure(name='child') platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) log.debug("Preparing a valid agent instance launch, for config only") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_child_obj) child_config = config_builder.p.prepare(will_launch=False) verify_child_config(child_config, platform_device_child_id) log.debug("Making the structure for a platform agent, which will be the parent") platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure(name='parent') platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id) log.debug("Testing child-less parent as a child config") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_child_config(parent_config, platform_device_parent_id) log.debug("assigning child platform to parent") assign_child_platform_to_parent_platform_fn(platform_device_child_id, platform_device_parent_id) child_device_ids = find_child_platform_ids_of_parent_platform_fn(platform_device_parent_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing parent + child as parent config") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj) parent_config = config_builder.p.prepare(will_launch=False) verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id) log.debug("making the structure for an instrument agent") instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure() instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id) log.debug("Testing instrument config") refresh_iconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.i.set_agent_instance_object(instrument_agent_instance_obj) instrument_config = config_builder.i.prepare(will_launch=False) verify_instrument_config(instrument_config, instrument_device_id) log.debug("assigning instrument to platform") self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id, platform_device_child_id) child_device_ids = self.RR2.find_instrument_device_ids_of_platform_device_using_has_device(platform_device_child_id) self.assertNotEqual(0, len(child_device_ids)) log.debug("Testing entire config") refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj) full_config = config_builder.p.prepare(will_launch=False) verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id) #self.fail(parent_config) #plauncher.prepare(will_launch=False) log.info("END base_agent_instance_config")
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.org_management_service = OrgManagementServiceClient( node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #print 'TestObservatoryManagementServiceIntegration: started services' self.event_publisher = EventPublisher() # @unittest.skip('this exists only for debugging the launch process') # def test_just_the_setup(self): # return def destroy(self, resource_ids): self.OMS.force_delete_observatory(resource_ids.observatory_id) self.OMS.force_delete_subsite(resource_ids.subsite_id) self.OMS.force_delete_subsite(resource_ids.subsite2_id) self.OMS.force_delete_subsite(resource_ids.subsiteb_id) self.OMS.force_delete_subsite(resource_ids.subsitez_id) self.OMS.force_delete_platform_site(resource_ids.platform_site_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id) self.OMS.force_delete_platform_site(resource_ids.platform_site3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id) self.OMS.force_delete_instrument_site( resource_ids.instrument_siteb3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id) #@unittest.skip('targeting') def test_observatory_management(self): resources = self._make_associations() self._do_test_find_related_sites(resources) self._do_test_get_sites_devices_status(resources) self._do_test_find_site_data_products(resources) self._do_test_find_related_frames_of_reference(resources) self._do_test_create_geospatial_point_center(resources) self._do_test_find_observatory_org(resources) self.destroy(resources) def _do_test_find_related_sites(self, resources): site_resources, site_children, _, _ = self.OMS.find_related_sites( resources.org_id) #import sys, pprint #print >> sys.stderr, pprint.pformat(site_resources) #print >> sys.stderr, pprint.pformat(site_children) #self.assertIn(resources.org_id, site_resources) self.assertIn(resources.observatory_id, site_resources) self.assertIn(resources.subsite_id, site_resources) self.assertIn(resources.subsite_id, site_resources) self.assertIn(resources.subsite2_id, site_resources) self.assertIn(resources.platform_site_id, site_resources) self.assertIn(resources.instrument_site_id, site_resources) self.assertEquals(len(site_resources), 13) self.assertEquals(site_resources[resources.observatory_id].type_, RT.Observatory) self.assertIn(resources.org_id, site_children) self.assertIn(resources.observatory_id, site_children) self.assertIn(resources.subsite_id, site_children) self.assertIn(resources.subsite_id, site_children) self.assertIn(resources.subsite2_id, site_children) self.assertIn(resources.platform_site_id, site_children) self.assertNotIn(resources.instrument_site_id, site_children) self.assertEquals(len(site_children), 9) self.assertIsInstance(site_children[resources.subsite_id], list) self.assertEquals(len(site_children[resources.subsite_id]), 2) def _do_test_get_sites_devices_status(self, resources): #bin/nosetests -s -v --nologcapture ion/services/sa/observatory/test/test_observatory_management_service_integration.py:TestObservatoryManagementServiceIntegration.test_observatory_management full_result_dict = self.OMS.get_sites_devices_status( parent_resource_ids=[resources.org_id], include_sites=True) result_dict = full_result_dict[resources.org_id] site_resources = result_dict.get("site_resources", None) site_children = result_dict.get("site_children", None) self.assertEquals(len(site_resources), 14) self.assertEquals(len(site_children), 9) full_result_dict = self.OMS.get_sites_devices_status( parent_resource_ids=[resources.org_id], include_sites=True, include_devices=True, include_status=True) result_dict = full_result_dict[resources.org_id] log.debug("RESULT DICT: %s", result_dict.keys()) site_resources = result_dict.get("site_resources", None) site_children = result_dict.get("site_children", None) site_status = result_dict.get("site_status", None) self.assertEquals(len(site_resources), 14) self.assertEquals(len(site_children), 9) full_result_dict = self.OMS.get_sites_devices_status( parent_resource_ids=[resources.observatory_id], include_sites=True, include_devices=True, include_status=True) result_dict = full_result_dict[resources.observatory_id] site_resources = result_dict.get("site_resources") site_children = result_dict.get("site_children") site_status = result_dict.get("site_status") self.assertEquals(len(site_resources), 13) self.assertEquals(len(site_children), 8) def _do_test_find_site_data_products(self, resources): res_dict = self.OMS.find_site_data_products(resources.org_id) #import sys, pprint #print >> sys.stderr, pprint.pformat(res_dict) self.assertIsNone(res_dict['data_product_resources']) self.assertIn(resources.platform_device_id, res_dict['device_data_products']) self.assertIn(resources.instrument_device_id, res_dict['device_data_products']) #@unittest.skip('targeting') def _do_test_find_related_frames_of_reference(self, stuff): # finding subordinates gives a dict of obj lists, convert objs to ids def idify(adict): ids = {} for k, v in adict.iteritems(): ids[k] = [] for obj in v: ids[k].append(obj._id) return ids # a short version of the function we're testing, with id-ify def short(resource_id, output_types): ret = self.OMS.find_related_frames_of_reference( resource_id, output_types) return idify(ret) #set up associations first stuff = self._make_associations() #basic traversal of tree from instrument to platform ids = short(stuff.instrument_site_id, [RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) #since this is the first search, just make sure the input inst_id got stripped if RT.InstrumentSite in ids: self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #basic traversal of tree from platform to instrument ids = short(stuff.platform_siteb_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite]) #full traversal of tree from observatory down to instrument ids = short(stuff.observatory_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #full traversal of tree from instrument to observatory ids = short(stuff.instrument_site_id, [RT.Observatory]) self.assertIn(RT.Observatory, ids) self.assertIn(stuff.observatory_id, ids[RT.Observatory]) #partial traversal, only down to platform ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(RT.InstrumentSite, ids) #partial traversal, only down to platform ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertNotIn(RT.Observatory, ids) self.destroy(stuff) def _make_associations(self): """ create one of each resource and association used by OMS to guard against problems in ion-definitions """ #raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41") """ the tree we're creating (observatory, sites, platforms, instruments) rows are lettered, colums numbered. - first row is implied a - first column is implied 1 - site Z, just because O--Sz | S--S2--P3--I4 | Sb-Pb2-Ib3 | P--I2 <- PlatformDevice, InstrumentDevice2 | Pb <- PlatformDevice b | I <- InstrumentDevice """ org_id = self.OMS.create_marine_facility(any_old(RT.Org)) def create_under_org(resource_type, extra_fields=None): obj = any_old(resource_type, extra_fields) if RT.InstrumentDevice == resource_type: resource_id = self.IMS.create_instrument_device(obj) else: resource_id, _ = self.RR.create(obj) self.OMS.assign_resource_to_observatory_org( resource_id=resource_id, org_id=org_id) return resource_id #stuff we control observatory_id = create_under_org(RT.Observatory) subsite_id = create_under_org(RT.Subsite) subsite2_id = create_under_org(RT.Subsite) subsiteb_id = create_under_org(RT.Subsite) subsitez_id = create_under_org(RT.Subsite) platform_site_id = create_under_org(RT.PlatformSite) platform_siteb_id = create_under_org(RT.PlatformSite) platform_siteb2_id = create_under_org(RT.PlatformSite) platform_site3_id = create_under_org(RT.PlatformSite) instrument_site_id = create_under_org(RT.InstrumentSite) instrument_site2_id = create_under_org(RT.InstrumentSite) instrument_siteb3_id = create_under_org(RT.InstrumentSite) instrument_site4_id = create_under_org(RT.InstrumentSite) #stuff we associate to instrument_device_id = create_under_org(RT.InstrumentDevice) instrument_device2_id = create_under_org(RT.InstrumentDevice) platform_device_id = create_under_org(RT.PlatformDevice) platform_deviceb_id = create_under_org(RT.PlatformDevice) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) deployment_id, _ = self.RR.create(any_old(RT.Deployment)) # marine tracking resources asset_id = create_under_org(RT.Asset) asset_type_id = create_under_org(RT.AssetType) event_duration_id = create_under_org(RT.EventDuration) event_duration_type_id = create_under_org(RT.EventDurationType) #observatory self.RR.create_association(observatory_id, PRED.hasSite, subsite_id) self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id) #site self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id) self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id) self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id) #platform_site(s) self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id) self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id) self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id) self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id) self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id) #test network parent link self.OMS.assign_device_to_network_parent(platform_device_id, platform_deviceb_id) self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id) #instrument_site(s) self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id) self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) ret = DotDict() ret.org_id = org_id ret.observatory_id = observatory_id ret.subsite_id = subsite_id ret.subsite2_id = subsite2_id ret.subsiteb_id = subsiteb_id ret.subsitez_id = subsitez_id ret.platform_site_id = platform_site_id ret.platform_siteb_id = platform_siteb_id ret.platform_siteb2_id = platform_siteb2_id ret.platform_site3_id = platform_site3_id ret.instrument_site_id = instrument_site_id ret.instrument_site2_id = instrument_site2_id ret.instrument_siteb3_id = instrument_siteb3_id ret.instrument_site4_id = instrument_site4_id ret.instrument_device_id = instrument_device_id ret.instrument_device2_id = instrument_device2_id ret.platform_device_id = platform_device_id ret.platform_deviceb_id = platform_deviceb_id ret.instrument_model_id = instrument_model_id ret.platform_model_id = platform_model_id ret.deployment_id = deployment_id ret.asset_id = asset_id ret.asset_type_id = asset_type_id ret.event_duration_id = event_duration_id ret.event_duration_type_id = event_duration_type_id return ret #@unittest.skip("targeting") def test_create_observatory(self): observatory_obj = IonObject(RT.Observatory, name='TestFacility', description='some new mf') observatory_id = self.OMS.create_observatory(observatory_obj) self.OMS.force_delete_observatory(observatory_id) #@unittest.skip("targeting") def _do_test_create_geospatial_point_center(self, resources): platformsite_obj = IonObject(RT.PlatformSite, name='TestPlatformSite', description='some new TestPlatformSite') geo_index_obj = IonObject(OT.GeospatialBounds) geo_index_obj.geospatial_latitude_limit_north = 20.0 geo_index_obj.geospatial_latitude_limit_south = 10.0 geo_index_obj.geospatial_longitude_limit_east = 15.0 geo_index_obj.geospatial_longitude_limit_west = 20.0 platformsite_obj.constraint_list = [geo_index_obj] platformsite_id = self.OMS.create_platform_site(platformsite_obj) # now get the dp back to see if it was updated platformsite_obj = self.OMS.read_platform_site(platformsite_id) self.assertEquals('some new TestPlatformSite', platformsite_obj.description) self.assertAlmostEqual(15.0, platformsite_obj.geospatial_point_center.lat, places=1) #now adjust a few params platformsite_obj.description = 'some old TestPlatformSite' geo_index_obj = IonObject(OT.GeospatialBounds) geo_index_obj.geospatial_latitude_limit_north = 30.0 geo_index_obj.geospatial_latitude_limit_south = 20.0 platformsite_obj.constraint_list = [geo_index_obj] update_result = self.OMS.update_platform_site(platformsite_obj) # now get the dp back to see if it was updated platformsite_obj = self.OMS.read_platform_site(platformsite_id) self.assertEquals('some old TestPlatformSite', platformsite_obj.description) self.assertAlmostEqual(25.0, platformsite_obj.geospatial_point_center.lat, places=1) self.OMS.force_delete_platform_site(platformsite_id) #@unittest.skip("targeting") def _do_test_find_observatory_org(self, resources): log.debug("Make TestOrg") org_obj = IonObject(RT.Org, name='TestOrg', description='some new mf org') org_id = self.OMS.create_marine_facility(org_obj) log.debug("Make Observatory") observatory_obj = IonObject(RT.Observatory, name='TestObservatory', description='some new obs') observatory_id = self.OMS.create_observatory(observatory_obj) log.debug("assign observatory to org") self.OMS.assign_resource_to_observatory_org(observatory_id, org_id) log.debug("verify assigment") org_objs = self.OMS.find_org_by_observatory(observatory_id) self.assertEqual(1, len(org_objs)) self.assertEqual(org_id, org_objs[0]._id) log.debug("org_id=<" + org_id + ">") log.debug("create a subsite with parent Observatory") subsite_obj = IonObject(RT.Subsite, name='TestSubsite', description='sample subsite') subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id) self.assertIsNotNone(subsite_id, "Subsite not created.") log.debug("verify that Subsite is linked to Observatory") mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id) self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.") log.debug("add the Subsite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id) log.debug("verify that Subsite is linked to Org") org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id) self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.") log.debug("create a logical platform with parent Subsite") platform_site_obj = IonObject(RT.PlatformSite, name='TestPlatformSite', description='sample logical platform') platform_site_id = self.OMS.create_platform_site( platform_site_obj, subsite_id) self.assertIsNotNone(platform_site_id, "PlatformSite not created.") log.debug("verify that PlatformSite is linked to Site") site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id) self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.") log.debug("add the PlatformSite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org( resource_id=platform_site_id, org_id=org_id) log.debug("verify that PlatformSite is linked to Org") org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id) self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.") log.debug("create a logical instrument with parent logical platform") instrument_site_obj = IonObject( RT.InstrumentSite, name='TestInstrumentSite', description='sample logical instrument') instrument_site_id = self.OMS.create_instrument_site( instrument_site_obj, platform_site_id) self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.") log.debug("verify that InstrumentSite is linked to PlatformSite") li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id) self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.") log.debug("add the InstrumentSite as a resource of this Observatory") self.OMS.assign_resource_to_observatory_org( resource_id=instrument_site_id, org_id=org_id) log.debug("verify that InstrumentSite is linked to Org") org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id) self.assertIsNotNone( org_li_assoc, "InstrumentSite not connected as resource to Org.") log.debug( "remove the InstrumentSite as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org( instrument_site_id, org_id) log.debug("verify that InstrumentSite is linked to Org") assocs, _ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True) self.assertEqual(0, len(assocs)) log.debug( "remove the InstrumentSite, association should drop automatically") self.OMS.delete_instrument_site(instrument_site_id) assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True) self.assertEqual(0, len(assocs)) log.debug("remove the PlatformSite as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org( platform_site_id, org_id) log.debug("verify that PlatformSite is linked to Org") assocs, _ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True) self.assertEqual(0, len(assocs)) log.debug("remove the Site as a resource of this Observatory") self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id) log.debug("verify that Site is linked to Org") assocs, _ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True) self.assertEqual(0, len(assocs)) self.RR.delete(org_id) self.OMS.force_delete_observatory(observatory_id) self.OMS.force_delete_subsite(subsite_id) self.OMS.force_delete_platform_site(platform_site_id) self.OMS.force_delete_instrument_site(instrument_site_id) @attr('EXT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side' ) def test_observatory_extensions(self): self.patch_cfg(CFG["container"], {"extended_resources": { "strip_results": False }}) obs_id = self.RR2.create(any_old(RT.Observatory)) pss_id = self.RR2.create( any_old(RT.PlatformSite, dict(alt_resource_type="StationSite"))) pas_id = self.RR2.create( any_old(RT.PlatformSite, dict(alt_resource_type="PlatformAssemblySite"))) pcs_id = self.RR2.create( any_old(RT.PlatformSite, dict(alt_resource_type="PlatformComponentSite"))) ins_id = self.RR2.create(any_old(RT.InstrumentSite)) obs_obj = self.RR2.read(obs_id) pss_obj = self.RR2.read(pss_id) pas_obj = self.RR2.read(pas_id) pcs_obj = self.RR2.read(pcs_id) ins_obj = self.RR2.read(ins_id) self.RR2.create_association(obs_id, PRED.hasSite, pss_id) self.RR2.create_association(pss_id, PRED.hasSite, pas_id) self.RR2.create_association(pas_id, PRED.hasSite, pcs_id) self.RR2.create_association(pcs_id, PRED.hasSite, ins_id) extended_obs = self.OMS.get_observatory_site_extension(obs_id, user_id=12345) self.assertEqual([pss_obj], extended_obs.platform_station_sites) self.assertEqual([pas_obj], extended_obs.platform_assembly_sites) self.assertEqual([pcs_obj], extended_obs.platform_component_sites) self.assertEqual([ins_obj], extended_obs.instrument_sites) extended_pss = self.OMS.get_observatory_site_extension(obs_id, user_id=12345) self.assertEqual([pas_obj], extended_pss.platform_assembly_sites) self.assertEqual([pcs_obj], extended_pss.platform_component_sites) self.assertEqual([ins_obj], extended_pss.instrument_sites) extended_pas = self.OMS.get_observatory_site_extension(pas_id, user_id=12345) self.assertEqual([pcs_obj], extended_pas.platform_component_sites) self.assertEqual([ins_obj], extended_pas.instrument_sites) extended_pcs = self.OMS.get_platform_component_site_extension( pcs_id, user_id=12345) self.assertEqual([ins_obj], extended_pcs.instrument_sites) #@unittest.skip("in development...") @attr('EXT') @attr('EXT1') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side' ) def test_observatory_org_extended(self): self.patch_cfg(CFG["container"], {"extended_resources": { "strip_results": False }}) stuff = self._make_associations() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product( input_resource_id=stuff.instrument_device_id, data_product_id=data_product_id1) #Create a user to be used as regular member member_actor_obj = IonObject(RT.ActorIdentity, name='org member actor') member_actor_id, _ = self.RR.create(member_actor_obj) assert (member_actor_id) member_actor_header = get_actor_header(member_actor_id) member_user_obj = IonObject(RT.UserInfo, name='org member user') member_user_id, _ = self.RR.create(member_user_obj) assert (member_user_id) self.RR.create_association(subject=member_actor_id, predicate=PRED.hasInfo, object=member_user_id) #Build the Service Agreement Proposal to enroll a user actor sap = IonObject(OT.EnrollmentProposal, consumer=member_actor_id, provider=stuff.org_id) sap_response = self.org_management_service.negotiate( sap, headers=member_actor_header) #enroll the member without using negotiation self.org_management_service.enroll_member(org_id=stuff.org_id, actor_id=member_actor_id) #-------------------------------------------------------------------------------- # Get the extended Site (platformSite) #-------------------------------------------------------------------------------- try: extended_site = self.OMS.get_site_extension(stuff.platform_site_id) except: log.error('failed to get extended site', exc_info=True) raise log.debug("extended_site: %r ", extended_site) self.assertEquals(stuff.subsiteb_id, extended_site.parent_site._id) self.assertEqual(2, len(extended_site.sites)) self.assertEqual(2, len(extended_site.platform_devices)) self.assertEqual(2, len(extended_site.platform_models)) self.assertIn(stuff.platform_device_id, [o._id for o in extended_site.platform_devices]) self.assertIn( stuff.platform_model_id, [o._id for o in extended_site.platform_models if o is not None]) log.debug( "verify that PlatformDeviceb is linked to PlatformDevice with hasNetworkParent link" ) associations = self.RR.find_associations( subject=stuff.platform_deviceb_id, predicate=PRED.hasNetworkParent, object=stuff.platform_device_id, id_only=True) self.assertIsNotNone( associations, "PlatformDevice child not connected to PlatformDevice parent.") #-------------------------------------------------------------------------------- # Get the extended Org #-------------------------------------------------------------------------------- #test the extended resource extended_org = self.OMS.get_marine_facility_extension(stuff.org_id) log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org)) #self.assertEqual(2, len(extended_org.instruments_deployed) ) #self.assertEqual(1, len(extended_org.platforms_not_deployed) ) self.assertEqual(2, extended_org.number_of_platforms) self.assertEqual(2, len(extended_org.platform_models)) self.assertEqual(2, extended_org.number_of_instruments) self.assertEqual(2, len(extended_org.instrument_models)) self.assertEqual(1, len(extended_org.members)) self.assertNotEqual(extended_org.members[0]._id, member_actor_id) self.assertEqual(extended_org.members[0]._id, member_user_id) self.assertEqual(1, len(extended_org.open_requests)) self.assertTrue(len(extended_site.deployments) > 0) self.assertEqual(len(extended_site.deployments), len(extended_site.deployment_info)) self.assertEqual(1, extended_org.number_of_assets) self.assertEqual(1, extended_org.number_of_asset_types) self.assertEqual(1, extended_org.number_of_event_durations) self.assertEqual(1, extended_org.number_of_event_duration_types) #test the extended resource of the ION org ion_org_id = self.org_management_service.find_org() extended_org = self.OMS.get_marine_facility_extension(ion_org_id._id, user_id=12345) log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org)) self.assertEqual(1, len(extended_org.members)) self.assertEqual(0, extended_org.number_of_platforms) #self.assertEqual(1, len(extended_org.sites)) #-------------------------------------------------------------------------------- # Get the extended Site #-------------------------------------------------------------------------------- #create device state events to use for op /non-op filtering in extended t = get_ion_ts() self.event_publisher.publish_event( ts_created=t, event_type='ResourceAgentStateEvent', origin=stuff.instrument_device_id, state=ResourceAgentState.STREAMING) self.event_publisher.publish_event( ts_created=t, event_type='ResourceAgentStateEvent', origin=stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE) extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id) log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site)) self.dpclient.delete_data_product(data_product_id1)
class TestInstrumentAlerts(IonIntegrationTestCase): pdict_id = None def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.catch_alert= gevent.queue.Queue() def _create_instrument_model(self): instModel_obj = IonObject( RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) self.addCleanup(self.imsclient.delete_instrument_model, instModel_id) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.4-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.addCleanup(self.imsclient.delete_instrument_agent, instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) self.addCleanup(self.imsclient.delete_instrument_device, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_stream_alarms(self, instDevice_id): #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ temp_alert_def = { 'name' : 'temperature_warning_interval', 'stream_name' : 'parsed', 'description' : 'Temperature is below the normal range of 50.0 and above.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'resource_id' : instDevice_id, 'origin_type' : 'device', 'lower_bound' : 50.0, 'lower_rel_op' : '<', 'alert_class' : 'IntervalAlert' } late_data_alert_def = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'description' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'value_id' : None, 'resource_id' : instDevice_id, 'origin_type' : 'device', 'time_delta' : 2, 'alert_class' : 'LateDataAlert' } return temp_alert_def, late_data_alert_def def _create_instrument_agent_instance(self, instAgent_id, instDevice_id): # port_agent_config = { # 'device_addr': CFG.device.sbe37.host, # 'device_port': CFG.device.sbe37.port, # 'process_type': PortAgentProcessType.UNIX, # 'binary_path': "port_agent", # 'port_agent_addr': 'localhost', # 'command_port': CFG.device.sbe37.port_agent_cmd_port, # 'data_port': CFG.device.sbe37.port_agent_data_port, # 'log_level': 5, # 'type': PortAgentType.ETHERNET # } port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': 4008, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } temp_alert, late_data_alert = self._create_instrument_stream_alarms(instDevice_id) instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config, alerts= [temp_alert, late_data_alert] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) self.addCleanup(self.imsclient.delete_instrument_agent_instance, instAgentInstance_id) return instAgentInstance_id def test_alerts(self): #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) # It is necessary for the instrument device to be associated with atleast one output data product tdom, sdom = time_series_domain() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) # We are creating two data products here, one for parsed and another raw dp_obj_parsed = IonObject(RT.DataProduct, name='parsed_data_product', description='Parsed output data product for instrument', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) dp_obj_raw = IonObject(RT.DataProduct, name='raw_data_prod', description='Raw output data product for instrument', temporal_domain = tdom.dump(), spatial_domain = sdom.dump()) parsed_out_data_prod_id = self.dataproductclient.create_data_product(data_product=dp_obj_parsed, stream_definition_id=parsed_stream_def_id) raw_out_data_prod_id = self.dataproductclient.create_data_product(data_product=dp_obj_raw, stream_definition_id=raw_stream_def_id) self.addCleanup(self.dataproductclient.delete_data_product, parsed_out_data_prod_id) self.addCleanup(self.dataproductclient.delete_data_product, raw_out_data_prod_id) self.dataproductclient.activate_data_product_persistence(data_product_id=parsed_out_data_prod_id) self.dataproductclient.activate_data_product_persistence(data_product_id=raw_out_data_prod_id) # todo: note that the generated config on the instruments will be done for both raw and parsed stream defs since these two data products constructed with each are associated as output data products with the instrument # todo: if the config is not generated for a stream def, then the instrument agent will complain if the simulator generates data corresponding to a stream def that is not there in the stream config as a mentioned stream def self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=parsed_out_data_prod_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=raw_out_data_prod_id) log.debug("assigned instdevice id: %s to data product: %s", instDevice_id, raw_out_data_prod_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id,instDevice_id ) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) # Wait for instrument agent to spawn gate = ProcessStateGate(self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Set up the subscriber to catch the alert event #------------------------------------------------------------------------------------- def callback_for_alert(event, *args, **kwargs): log.debug("caught an alert: %s", event) self.catch_alert.put(event) self.event_subscriber = EventSubscriber(event_type='DeviceStatusAlertEvent', origin=instDevice_id, callback=callback_for_alert) self.event_subscriber.start() self.addCleanup(self.event_subscriber.stop) #------------------------------------------------------------------------------------- # Running the instrument.... #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) got_bad_temp = False got_late_data = False runtime = 0 starttime = time.time() caught_events = [] while (got_bad_temp == False or got_late_data == False) and \ runtime < 120: a = self.catch_alert.get(timeout=90) caught_events.append(a) if a.name == 'temperature_warning_interval' and \ a.description == 'Temperature is below the normal range of 50.0 and above.': got_bad_temp = True if a.name == 'late_data_warning' and \ a.description == 'Expected data has not arrived.': got_late_data = True runtime = time.time() - starttime log.debug("caught_events: %s", [c.name for c in caught_events]) for c in caught_events: self.assertIn(c.name, ['temperature_warning_interval', 'late_data_warning']) self.assertEqual(c.origin, instDevice_id) self.assertEqual(c.type_, 'DeviceStatusAlertEvent') self.assertEqual(c.origin_type, 'InstrumentDevice') self.assertTrue(got_bad_temp) self.assertTrue(got_late_data)
class TestWorkflowManagementIntegration(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.workflowclient = WorkflowManagementServiceClient( node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient( node=self.container.node) self.ctd_stream_def = SBE37_CDM_stream_definition() def _create_ctd_input_stream_and_data_product( self, data_product_name='ctd_parsed'): cc = self.container assertions = self.assertTrue #------------------------------- # Create CTD Parsed as the initial data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition( container=self.ctd_stream_def, name='Simulated CTD data') log.debug('Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name=data_product_name, description='ctd stream test') try: ctd_parsed_data_product_id = self.dataproductclient.create_data_product( dp_obj, ctd_stream_def_id) except Exception as ex: self.fail("failed to create new data product: %s" % ex) log.debug('new ctd_parsed_data_product_id = ', ctd_parsed_data_product_id) #Only ever need one device for testing purposes. instDevice_obj, _ = self.rrclient.find_resources( restype=RT.InstrumentDevice, name='SBE37IMDevice') if instDevice_obj: instDevice_id = instDevice_obj[0]._id else: instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product_id) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_parsed_data_product_id, persist_data=False, persist_metadata=False) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) ctd_stream_id = stream_ids[0] return ctd_stream_id, ctd_parsed_data_product_id def _start_simple_input_stream_process(self, ctd_stream_id): return self._start_input_stream_process(ctd_stream_id) def _start_sinusoidal_input_stream_process(self, ctd_stream_id): return self._start_input_stream_process( ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher', 'SinusoidalCtdPublisher') def _start_input_stream_process( self, ctd_stream_id, module='ion.processes.data.ctd_stream_publisher', class_name='SimpleCtdPublisher'): ### ### Start the process for producing the CTD data ### # process definition for the ctd simulator... producer_definition = ProcessDefinition() producer_definition.executable = { 'module': module, 'class': class_name } ctd_sim_procdef_id = self.process_dispatcher.create_process_definition( process_definition=producer_definition) # Start the ctd simulator to produce some data configuration = { 'process': { 'stream_id': ctd_stream_id, } } ctd_sim_pid = self.process_dispatcher.schedule_process( process_definition_id=ctd_sim_procdef_id, configuration=configuration) return ctd_sim_pid def _start_output_stream_listener(self, data_product_stream_ids, message_count_per_stream=10): cc = self.container assertions = self.assertTrue ### ### Make a subscriber in the test to listen for transformed data ### salinity_subscription_id = self.pubsubclient.create_subscription( query=StreamQuery(data_product_stream_ids), exchange_name='workflow_test', name="test workflow transformations", ) pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node) result = gevent.event.AsyncResult() results = [] def message_received(message, headers): # Heads log.warn(' data received!') results.append(message) if len(results) >= len( data_product_stream_ids ) * message_count_per_stream: #Only wait for so many messages - per stream result.set(True) subscriber = subscriber_registrar.create_subscriber( exchange_name='workflow_test', callback=message_received) subscriber.start() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id) # Assert that we have received data assertions(result.get(timeout=30)) self.pubsubclient.deactivate_subscription( subscription_id=salinity_subscription_id) subscriber.stop() return results def _validate_messages(self, results): cc = self.container assertions = self.assertTrue first_salinity_values = None for message in results: try: psd = PointSupplementStreamParser( stream_definition=self.ctd_stream_def, stream_granule=message) temp = psd.get_values('temperature') log.info(psd.list_field_names()) except KeyError as ke: temp = None if temp is not None: assertions(isinstance(temp, numpy.ndarray)) log.info('temperature=' + str(numpy.nanmin(temp))) first_salinity_values = None else: psd = PointSupplementStreamParser( stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message) log.info(psd.list_field_names()) # Test the handy info method for the names of fields in the stream def assertions('salinity' in psd.list_field_names()) # you have to know the name of the coverage in stream def salinity = psd.get_values('salinity') log.info('salinity=' + str(numpy.nanmin(salinity))) assertions(isinstance(salinity, numpy.ndarray)) assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0 if first_salinity_values is None: first_salinity_values = salinity.tolist() else: second_salinity_values = salinity.tolist() assertions( len(first_salinity_values) == len( second_salinity_values)) for idx in range(0, len(first_salinity_values)): assertions(first_salinity_values[idx] * 2.0 == second_salinity_values[idx]) def _create_salinity_data_process_definition(self): # Salinity: Data Process Definition #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='ctd_salinity') if len(dpd) > 0: return dpd[0] log.debug("Create data process definition SalinityTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_salinity', description='create a salinity data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Excpetion as ex: self.fail( "failed to create new SalinityTransform data process definition: %s" % ex) # create a stream definition for the data from the salinity Transform sal_stream_def_id = self.pubsubclient.create_stream_definition( container=SalinityTransform.outgoing_stream_def, name='Salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition( sal_stream_def_id, ctd_L2_salinity_dprocdef_id) return ctd_L2_salinity_dprocdef_id def _create_salinity_doubler_data_process_definition(self): #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler') if len(dpd) > 0: return dpd[0] # Salinity Doubler: Data Process Definition log.debug("Create data process definition SalinityDoublerTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='salinity_doubler', description='create a salinity doubler data product', module='ion.processes.data.transforms.example_double_salinity', class_name='SalinityDoubler', process_source='SalinityDoubler source code here...') try: salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Exception as ex: self.fail( "failed to create new SalinityDoubler data process definition: %s" % ex) # create a stream definition for the data from the salinity Transform salinity_double_stream_def_id = self.pubsubclient.create_stream_definition( container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler') self.dataprocessclient.assign_stream_definition_to_data_process_definition( salinity_double_stream_def_id, salinity_doubler_dprocdef_id) return salinity_doubler_dprocdef_id def create_transform_process(self, data_process_definition_id, data_process_input_dp_id): data_process_definition = self.rrclient.read( data_process_definition_id) # Find the link between the output Stream Definition resource and the Data Process Definition resource stream_ids, _ = self.rrclient.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True) if not stream_ids: raise Inconsistent( "The data process definition %s is missing an association to an output stream definition" % data_process_definition._id) process_output_stream_def_id = stream_ids[0] #Concatenate the name of the workflow and data process definition for the name of the data product output data_process_name = data_process_definition.name # Create the output data product of the transform transform_dp_obj = IonObject( RT.DataProduct, name=data_process_name, description=data_process_definition.description) transform_dp_id = self.dataproductclient.create_data_product( transform_dp_obj, process_output_stream_def_id) self.dataproductclient.activate_data_product_persistence( data_product_id=transform_dp_id, persist_data=True, persist_metadata=True) #last one out of the for loop is the output product id output_data_product_id = transform_dp_id # Create the transform data process log.debug("create data_process and start it") data_process_id = self.dataprocessclient.create_data_process( data_process_definition._id, [data_process_input_dp_id], {'output': transform_dp_id}) self.dataprocessclient.activate_data_process(data_process_id) #Find the id of the output data stream stream_ids, _ = self.rrclient.find_objects(transform_dp_id, PRED.hasStream, None, True) if not stream_ids: raise Inconsistent( "The data process %s is missing an association to an output stream" % data_process_id) return data_process_id, output_data_product_id @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #@unittest.skip("Skipping for debugging ") def test_SA_transform_components(self): assertions = self.assertTrue #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) ### ### Setup the first transformation ### # Salinity: Data Process Definition ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition( ) l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process( ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product_id) ## get the stream id for the transform outputs stream_ids, _ = self.rrclient.find_objects( ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) sal_stream_id = stream_ids[0] data_product_stream_ids.append(sal_stream_id) ### ### Setup the second transformation ### # Salinity Doubler: Data Process Definition salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition( ) salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process( salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id) stream_ids, _ = self.rrclient.find_objects( salinity_doubler_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) sal_dbl_stream_id = stream_ids[0] data_product_stream_ids.append(sal_dbl_stream_id) #Start the input stream process ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id) #Start te output stream listener to monitor and verify messages results = self._start_output_stream_listener(data_product_stream_ids) #Stop the transform processes self.dataprocessclient.deactivate_data_process( salinity_double_data_process_id) self.dataprocessclient.deactivate_data_process( l2_salinity_all_data_process_id) # stop the flow parse the messages... self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Validate the data from each of the messages along the way self._validate_messages(results) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #@unittest.skip("Skipping for debugging ") def test_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Salinity_Test_Workflow', description='tests a workflow of multiple transform data processes' ) workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #Add a transformation process definition ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False ) #Don't persist the intermediate data product workflow_def_obj.workflow_steps.append(workflow_step_obj) #Add a transformation process definition salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=salinity_doubler_dprocdef_id, output_data_product_name=workflow_data_product_name) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 2) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, timeout=30) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Verify the output data product name matches what was specified in the workflow definition workflow_product = self.rrclient.read(workflow_product_id) assertions(workflow_product.name == workflow_data_product_name) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 2) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the input stream process ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id) #Start the output stream listener to monitor and verify messages results = self._start_output_stream_listener(data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id, False, timeout=15) # Should test true at some point #Make sure the Workflow object was removed objs, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(objs) == 0) # stop the flow parse the messages... self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Validate the data from each of the messages along the way self._validate_messages(results) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids, _ = self.rrclient.find_resources( restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0) def _create_google_dt_data_process_definition(self): #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform') if len(dpd) > 0: return dpd[0] # Data Process Definition log.debug("Create data process definition GoogleDtTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='google_dt_transform', description='Convert data streams to Google DataTables', module='ion.processes.data.transforms.viz.google_dt', class_name='VizTransformGoogleDT', process_source='VizTransformGoogleDT source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Exception as ex: self.fail( "failed to create new VizTransformGoogleDT data process definition: %s" % ex) # create a stream definition for the data from the stream_def_id = self.pubsubclient.create_stream_definition( container=VizTransformGoogleDT.outgoing_stream_def, name='VizTransformGoogleDT') self.dataprocessclient.assign_stream_definition_to_data_process_definition( stream_def_id, procdef_id) return procdef_id def _validate_google_dt_results(self, results_stream_def, results): cc = self.container assertions = self.assertTrue for g in results: if isinstance(g, Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) #log.warn(tx.pretty_print()) #log.warn(rdt.pretty_print()) gdt_component = rdt['google_dt_components'][0] assertions( gdt_component['viz_product_type'] == 'google_realtime_dt') gdt_description = gdt_component['data_table_description'] gdt_content = gdt_component['data_table_content'] assertions(gdt_description[0][0] == 'time') assertions(len(gdt_description) > 1) assertions(len(gdt_content) >= 0) return def _create_mpl_graphs_data_process_definition(self): #First look to see if it exists and if not, then create it dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform') if len(dpd) > 0: return dpd[0] #Data Process Definition log.debug("Create data process definition MatplotlibGraphsTransform") dpd_obj = IonObject( RT.DataProcessDefinition, name='mpl_graphs_transform', description='Convert data streams to Matplotlib graphs', module='ion.processes.data.transforms.viz.matplotlib_graphs', class_name='VizTransformMatplotlibGraphs', process_source='VizTransformMatplotlibGraphs source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except Exception as ex: self.fail( "failed to create new VizTransformMatplotlibGraphs data process definition: %s" % ex) # create a stream definition for the data stream_def_id = self.pubsubclient.create_stream_definition( container=VizTransformMatplotlibGraphs.outgoing_stream_def, name='VizTransformMatplotlibGraphs') self.dataprocessclient.assign_stream_definition_to_data_process_definition( stream_def_id, procdef_id) return procdef_id def _validate_mpl_graphs_results(self, results_stream_def, results): cc = self.container assertions = self.assertTrue for g in results: if isinstance(g, Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) #log.warn(tx.pretty_print()) #log.warn(rdt.pretty_print()) graphs = rdt['matplotlib_graphs'] for graph in graphs: assertions( graph['viz_product_type'] == 'matplotlib_graphs') # check to see if the list (numpy array) contians actual images assertions( imghdr.what(graph['image_name'], graph['image_obj']) == 'png') return @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #unittest.skip("Skipping for debugging ") def test_google_dt_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='GoogleDT_Test_Workflow', description= 'Tests the workflow of converting stream data to Google DT') #Add a transformation process definition google_dt_procdef_id = self._create_google_dt_data_process_definition() workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the input stream process ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id) #Start the output stream listener to monitor and verify messages results = self._start_output_stream_listener(data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id, False) # Should test true at some point # stop the flow parse the messages... self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Validate the data from each of the messages along the way self._validate_google_dt_results( VizTransformGoogleDT.outgoing_stream_def, results) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids, _ = self.rrclient.find_resources( restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #@unittest.skip("Skipping for debugging ") def test_mpl_graphs_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Mpl_Graphs_Test_Workflow', description= 'Tests the workflow of converting stream data to Matplotlib graphs' ) #Add a transformation process definition mpl_graphs_procdef_id = self._create_mpl_graphs_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=mpl_graphs_procdef_id) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the input stream process ctd_sim_pid = self._start_sinusoidal_input_stream_process( ctd_stream_id) #Start the output stream listener to monitor and verify messages results = self._start_output_stream_listener(data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id, False) # Should test true at some point # stop the flow parse the messages... self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Validate the data from each of the messages along the way self._validate_mpl_graphs_results( VizTransformGoogleDT.outgoing_stream_def, results) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids, _ = self.rrclient.find_resources( restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') #@unittest.skip("Skipping for debugging ") def test_multiple_workflow_instances(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Multiple_Test_Workflow', description='Tests the workflow of converting stream data') #Add a transformation process definition google_dt_procdef_id = self._create_google_dt_data_process_definition() workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the first input data product ctd_stream_id1, ctd_parsed_data_product_id1 = self._create_ctd_input_stream_and_data_product( 'ctd_parsed1') data_product_stream_ids.append(ctd_stream_id1) #Create and start the first workflow workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id1) #Create the second input data product ctd_stream_id2, ctd_parsed_data_product_id2 = self._create_ctd_input_stream_and_data_product( 'ctd_parsed2') data_product_stream_ids.append(ctd_stream_id2) #Create and start the first workflow workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id2) #Walk the associations to find the appropriate output data streams to validate the messages workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 2) #Start the first input stream process ctd_sim_pid1 = self._start_sinusoidal_input_stream_process( ctd_stream_id1) #Start the second input stream process ctd_sim_pid2 = self._start_simple_input_stream_process(ctd_stream_id2) #Start the output stream listener to monitor a set number of messages being sent through the workflows results = self._start_output_stream_listener( data_product_stream_ids, message_count_per_stream=5) # stop the flow of messages... self.process_dispatcher.cancel_process( ctd_sim_pid1 ) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid2) #Stop the first workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id1, False) # Should test true at some point #Stop the second workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id2, False) # Should test true at some point workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 0) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids, _ = self.rrclient.find_resources( restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0) aid_list = self.rrclient.find_associations( workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aid_list) == 0)
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient self.processdispatchclient = ProcessDispatcherServiceClient(node = self.container.node) def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.info('test_createTransformsThenActivateInstrument: new InstrumentModel id = %s', instModel_id) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s', instDevice_id) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance" ) self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) log.info( 'TestDataProcessWithLookupTable: new Stream Definition id = %s', instDevice_id) log.info( 'Creating new CDM data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) log.info( 'new ctd_parsed_data_product_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.info('TestDataProcessWithLookupTable: Data product streams1 = %s', stream_ids) #------------------------------- # Create CTD Raw as the second data product #------------------------------- log.info('TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition') raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='ctd_raw', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) log.info( 'new ctd_raw_data_product_id = %s', ctd_raw_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.info( 'Data product streams2 = %s', stream_ids) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) ) processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment) log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) ) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) out_prods = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id,ctd_l0_temperature_output_dp_id] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L0_all_dprocdef_id, in_data_product_ids = in_prods, out_data_product_ids = out_prods) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("TestDataProcessWithLookupTable: create L0 all data_process return") data_process= self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject= ctd_l0_all_data_process_id, predicate= PRED.hasProcess, object_type= RT.Process,id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s', processAttachment)
class VisStreamLauncher(ImmediateProcess): """ Class emulates a stream source from a NetCDF file. It emits a record of data every few seconds on a stream identified by a routing key. """ def on_init(self): log.debug("VizStreamProducer init. Self.id=%s" % self.id) def on_start(self): log.debug("VizStreamProducer start") self.data_source_name = self.CFG.get_safe('name', 'sine_wave_generator') self.dataset = self.CFG.get_safe('dataset', 'sinusoidal') # create a pubsub client and a resource registry client self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) # Dummy instrument related clients self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() # create the pubsub client self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) # Additional code for creating a dummy instrument """ # Set up the preconditions. Look for an existing ingestion config while True: log.info("VisStreamLauncher:on_start: Waiting for an ingestion configuration to be available.") ingestion_cfgs, _ = self.rrclient.find_resources(RT.IngestionConfiguration, None, None, True) if len(ingestion_cfgs) > 0: break else: gevent.sleep(1) """ # Check to see if the data_product already exists in the system (for e.g re launching the code after a crash) dp_ids, _ = self.rrclient.find_resources(RT.DataProduct, None, self.data_source_name, True) if len(dp_ids) > 0: data_product_id = dp_ids[0] else: # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name=self.data_source_name, description=self.data_source_name) instModel_id = self.imsclient.create_instrument_model( instModel_obj) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name=self.data_source_name, description=self.data_source_name, serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition( name="SBE37_CDM", description="SBE37_CDM", parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name=self.data_source_name, description='Example ctd stream', temporal_domain=tdom, spatial_domain=sdom) data_product_id = self.dpclient.create_data_product( dp_obj, stream_definition_id=ctd_stream_def_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=data_product_id) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id) print ">>>>>>>>>>>>> Dataproduct for sine wave generator : ", data_product_id # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id, PRED.hasStream, None, True) if self.dataset == 'sinusoidal': self.container.spawn_process( name='ctd_test.' + self.data_source_name, module='ion.processes.data.sinusoidal_stream_publisher', cls='SinusoidalCtdPublisher', config={'process': { 'stream_id': stream_ids[0] }}) else: self.container.spawn_process( name='ctd_test.' + self.data_source_name, module='ion.processes.data.ctd_stream_publisher', cls='SimpleCtdPublisher', config={'process': { 'stream_id': stream_ids[0] }}) """ workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition, name='Realtime_HighCharts', id_only=True) if not len(workflow_def_ids): helper_create_highcharts_workflow_def(self.container) """ def on_quit(self): super(VisStreamLauncher, self).on_quit() log.debug("VizStreamProducer quit")
class TestPlatformInstrument(BaseIntTestPlatform): def setUp(self): self._start_container() self._pp = pprint.PrettyPrinter() log.debug("oms_uri = %s", OMS_URI) self.oms = CIOMSClientFactory.create_instance(OMS_URI) #url = OmsTestMixin.start_http_server() #log.debug("TestPlatformInstrument:setup http url %s", url) # #result = self.oms.event.register_event_listener(url) #log.debug("TestPlatformInstrument:setup register_event_listener result %s", result) self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.org_id = self.RR2.create(any_old(RT.Org)) log.debug("Org created: %s", self.org_id) # see _set_receive_timeout self._receive_timeout = 300 self.instrument_device_id = '' self.platform_device_id = '' self.platform_site_id = '' self.platform_agent_instance_id = '' self._pa_client = '' def done(): CIOMSClientFactory.destroy_instance(self.oms) event_notifications = OmsTestMixin.stop_http_server() log.info("event_notifications = %s" % str(event_notifications)) self.addCleanup(done) @unittest.skip('Must be run locally...') def test_platform_with_instrument_streaming(self): # # The following is with just a single platform and the single # instrument "SBE37_SIM_08", which corresponds to the one on port 4008. # #load the paramaters and the param dicts necesssary for the VEL3D log.debug( "load params------------------------------------------------------------------------------") self._load_params() log.debug( " _register_oms_listener------------------------------------------------------------------------------") self._register_oms_listener() #create the instrument device/agent/mode log.debug( "---------- create_instrument_resources ----------" ) self._create_instrument_resources() #create the platform device, agent and instance log.debug( "---------- create_platform_configuration ----------" ) self._create_platform_configuration('LPJBox_CI_Ben_Hall') self.rrclient.create_association(subject=self.platform_device_id, predicate=PRED.hasDevice, object=self.instrument_device_id) log.debug( "---------- start_platform ----------" ) self._start_platform() self.addCleanup(self._stop_platform) # get everything in command mode: self._ping_agent() log.debug( " ---------- initialize ----------" ) self._initialize() _ia_client = ResourceAgentClient(self.instrument_device_id, process=FakeProcess()) state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) log.debug( " ---------- go_active ----------" ) self._go_active() state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) log.debug( "---------- run ----------" ) self._run() gevent.sleep(2) log.debug( " ---------- _start_resource_monitoring ----------" ) self._start_resource_monitoring() gevent.sleep(2) # # # verify the instrument is command state: # state = ia_client.get_agent_state() # log.debug(" TestPlatformInstrument get_agent_state: %s", state) # self.assertEqual(state, ResourceAgentState.COMMAND) _stop_resource_monitoring log.debug( " ---------- _stop_resource_monitoring ----------" ) self._stop_resource_monitoring() gevent.sleep(2) log.debug( " ---------- go_inactive ----------" ) self._go_inactive() state = _ia_client.get_agent_state() log.info("TestPlatformInstrument get_agent_state %s", state) self._reset() self._shutdown() def _get_platform_attributes(self): log.debug( " ----------get_platform_attributes ----------") attr_infos = self.oms.attr.get_platform_attributes('LPJBox_CI_Ben_Hall') log.debug('_get_platform_attributes: %s', self._pp.pformat(attr_infos)) attrs = attr_infos['LPJBox_CI_Ben_Hall'] for attrid, arrinfo in attrs.iteritems(): arrinfo['attr_id'] = attrid log.debug('_get_platform_attributes: %s', self._pp.pformat(attrs)) return attrs def _load_params(self): log.info(" ---------- load_params ----------") # load_parameter_scenarios self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories="ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) def _create_platform_configuration(self, platform_id, parent_platform_id=None): """ This method is an adaptation of test_agent_instance_config in test_instrument_management_service_integration.py @param platform_id @param parent_platform_id @return a DotDict with various of the constructed elements associated to the platform. """ tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() param_dict_name = 'platform_eng_parsed' parsed_rpdict_id = self.dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) self.parsed_stream_def_id = self.pubsubclient.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_rpdict_id) driver_config = PLTFRM_DVR_CONFIG driver_config['attributes'] = self._get_platform_attributes() #self._platform_attributes[platform_id] #OMS returning an error for port.get_platform_ports #driver_config['ports'] = self._platform_ports[platform_id] log.debug("driver_config: %s", driver_config) # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, { 'driver_config': driver_config}) platform_agent_instance_obj.agent_config = { 'platform_config': { 'platform_id': 'LPJBox_CI_Ben_Hall', 'parent_platform_id': None } } self.platform_agent_instance_id = self.imsclient.create_platform_agent_instance(platform_agent_instance_obj) # agent creation platform_agent_obj = any_old(RT.PlatformAgent, { "stream_configurations": self._get_platform_stream_configs(), 'driver_module': PLTFRM_DVR_MOD, 'driver_class': PLTFRM_DVR_CLS}) platform_agent_id = self.imsclient.create_platform_agent(platform_agent_obj) # device creation self.platform_device_id = self.imsclient.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id) self.damsclient.assign_data_product(input_resource_id=self.platform_device_id, data_product_id=dp_id) self.dpclient.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.dpclient.delete_data_product, dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(self.platform_agent_instance_id, self.platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, self.platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(self.platform_agent_instance_id, self.org_id) ####################################### # dataset log.debug('data product = %s', dp_id) stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) log.debug('Data product stream_ids = %s', stream_ids) stream_id = stream_ids[0] # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) ####################################### log.debug('_create_platform_site_and_deployment platform_device_id: %s', self.platform_device_id) site_object = IonObject(RT.PlatformSite, name='PlatformSite1') self.platform_site_id = self.omsclient.create_platform_site(platform_site=site_object, parent_id='') log.debug('_create_platform_site_and_deployment site id: %s', self.platform_site_id) #create supporting objects for the Deployment resource # 1. temporal constraint # find current deployment using time constraints current_time = int( calendar.timegm(time.gmtime()) ) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) # 2. PlatformPort object which defines device to port map platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-01-CTDMO0999', port_type=PortTypeEnum.UPLINK, ip_address='0') # now create the Deployment deployment_obj = IonObject(RT.Deployment, name='TestPlatformDeployment', description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds], port_assignments={self.platform_device_id:platform_port_obj}) platform_deployment_id = self.omsclient.create_deployment(deployment=deployment_obj, site_id=self.platform_site_id, device_id=self.platform_device_id) log.debug('_create_platform_site_and_deployment deployment_id: %s', platform_deployment_id) deploy_obj2 = self.omsclient.read_deployment(platform_deployment_id) log.debug('_create_platform_site_and_deployment deploy_obj2 : %s', deploy_obj2) return self.platform_site_id, platform_deployment_id def _create_instrument_resources(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='VEL3D', description="VEL3D") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw' ) vel3d_b_sample = StreamConfiguration(stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample') vel3d_b_engineering = StreamConfiguration(stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering') # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg", stream_configurations = [raw_config, vel3d_b_sample, vel3d_b_engineering]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name='VEL3DDevice', description="VEL3DDevice", serial_number="12345" ) self.instrument_device_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, self.instrument_device_id) port_agent_config = { 'device_addr': '10.180.80.6', 'device_port': 2101, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 1025, 'data_port': 1026, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='VEL3DAgentInstance', description="VEL3DAgentInstance", port_agent_config = port_agent_config, alerts= []) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, self.instrument_device_id) self._start_port_agent(self.imsclient.read_instrument_agent_instance(instAgentInstance_id)) vel3d_b_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_sample', id_only=True) vel3d_b_sample_stream_def_id = self.pubsubclient.create_stream_definition(name='vel3d_b_sample', parameter_dictionary_id=vel3d_b_sample_pdict_id) vel3d_b_engineering_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_engineering', id_only=True) vel3d_b_engineering_stream_def_id = self.pubsubclient.create_stream_definition(name='vel3d_b_engineering', parameter_dictionary_id=vel3d_b_engineering_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='vel3d_b_sample', description='vel3d_b_sample', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=vel3d_b_sample_stream_def_id) self.damsclient.assign_data_product(input_resource_id=self.instrument_device_id, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) dp_obj = IonObject(RT.DataProduct, name='vel3d_b_engineering', description='vel3d_b_engineering', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=vel3d_b_engineering_stream_def_id) self.damsclient.assign_data_product(input_resource_id=self.instrument_device_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id3 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=self.instrument_device_id, data_product_id=data_product_id3) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id3) #create instrument site and associated deployment site_object = IonObject(RT.InstrumentSite, name='InstrumentSite1') instrument_site_id = self.omsclient.create_instrument_site(instrument_site=site_object, parent_id=self.platform_site_id) log.debug('_create_instrument_site_and_deployment site id: %s', instrument_site_id) #create supporting objects for the Deployment resource # 1. temporal constraint # find current deployment using time constraints current_time = int( calendar.timegm(time.gmtime()) ) # two years on either side of current time start = current_time - 63115200 end = current_time + 63115200 temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end)) # 2. PlatformPort object which defines device to port map platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-03-CTDMO0999', port_type=PortTypeEnum.PAYLOAD, ip_address='0') # now create the Deployment deployment_obj = IonObject(RT.Deployment, name='TestInstrumentDeployment', description='some new deployment', context=IonObject(OT.CabledInstrumentDeploymentContext), constraint_list=[temporal_bounds], port_assignments={self.instrument_device_id:platform_port_obj}) instrument_deployment_id = self.omsclient.create_deployment(deployment=deployment_obj, site_id=instrument_site_id, device_id=self.instrument_device_id) log.debug('_create_instrument_site_and_deployment deployment_id: %s', instrument_deployment_id) def _start_port_agent(self, instrument_agent_instance_obj=None): """ Construct and start the port agent, ONLY NEEDED FOR INSTRUMENT AGENTS. """ _port_agent_config = instrument_agent_instance_obj.port_agent_config # It blocks until the port agent starts up or a timeout _pagent = PortAgentProcess.launch_process(_port_agent_config, test_mode = True) pid = _pagent.get_pid() port = _pagent.get_data_port() cmd_port = _pagent.get_command_port() log.info("IMS:_start_pagent returned from PortAgentProcess.launch_process pid: %s ", pid) # Hack to get ready for DEMO. Further though needs to be put int # how we pass this config info around. host = 'localhost' driver_config = instrument_agent_instance_obj.driver_config comms_config = driver_config.get('comms_config') if comms_config: host = comms_config.get('addr') else: log.warn("No comms_config specified, using '%s'" % host) # Configure driver to use port agent port number. instrument_agent_instance_obj.driver_config['comms_config'] = { 'addr' : host, 'cmd_port' : cmd_port, 'port' : port } instrument_agent_instance_obj.driver_config['pagent_pid'] = pid self.imsclient.update_instrument_agent_instance(instrument_agent_instance_obj) return self.imsclient.read_instrument_agent_instance(instrument_agent_instance_obj._id) def _start_platform(self): """ Starts the given platform waiting for it to transition to the UNINITIALIZED state (note that the agent starts in the LAUNCHING state). More in concrete the sequence of steps here are: - prepares subscriber to receive the UNINITIALIZED state transition - launches the platform process - waits for the start of the process - waits for the transition to the UNINITIALIZED state """ ############################################################## # prepare to receive the UNINITIALIZED state transition: async_res = AsyncResult() def consume_event(evt, *args, **kwargs): log.debug("Got ResourceAgentStateEvent %s from origin %r", evt.state, evt.origin) if evt.state == PlatformAgentState.UNINITIALIZED: async_res.set(evt) # start subscriber: sub = EventSubscriber(event_type="ResourceAgentStateEvent", origin=self.platform_device_id, callback=consume_event) sub.start() log.info("registered event subscriber to wait for state=%r from origin %r", PlatformAgentState.UNINITIALIZED, self.platform_device_id) #self._event_subscribers.append(sub) sub._ready_event.wait(timeout=EVENT_TIMEOUT) ############################################################## # now start the platform: agent_instance_id = self.platform_agent_instance_id log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id) pid = self.imsclient.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id) log.debug("start_platform_agent_instance returned pid=%s", pid) #wait for start agent_instance_obj = self.imsclient.read_platform_agent_instance(agent_instance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, self.platform_device_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds") # Start a resource agent client to talk with the agent. self._pa_client = ResourceAgentClient(self.platform_device_id, name=gate.process_id, process=FakeProcess()) log.debug("got platform agent client %s", str(self._pa_client)) ############################################################## # wait for the UNINITIALIZED event: async_res.get(timeout=self._receive_timeout) def _register_oms_listener(self): #load the paramaters and the param dicts necesssary for the VEL3D log.debug( "---------- connect_to_oms ---------- ") log.debug("oms_uri = %s", OMS_URI) self.oms = CIOMSClientFactory.create_instance(OMS_URI) #buddha url url = "http://10.22.88.168:5000/ion-service/oms_event" log.info("test_oms_events_receive:setup http url %s", url) result = self.oms.event.register_event_listener(url) log.debug("_register_oms_listener register_event_listener result %s", result) #------------------------------------------------------------------------------------- # Set up the subscriber to catch the alert event #------------------------------------------------------------------------------------- def callback_for_alert(event, *args, **kwargs): log.debug("caught an OMSDeviceStatusEvent: %s", event) self.catch_alert.put(event) self.event_subscriber = EventSubscriber(event_type='OMSDeviceStatusEvent', callback=callback_for_alert) self.event_subscriber.start() self.addCleanup(self.event_subscriber.stop) result = self.oms.event.generate_test_event({'platform_id': 'fake_platform_id', 'message': "fake event triggered from CI using OMS' generate_test_event", 'severity': '3', 'group ': 'power'}) log.debug("_register_oms_listener generate_test_event result %s", result) def _stop_platform(self): try: self.IMS.stop_platform_agent_instance(self.platform_agent_instance_id) except Exception: log.warn( "platform_id=%r: Exception in IMS.stop_platform_agent_instance with " "platform_agent_instance_id = %r. Perhaps already dead.", self.platform_device_id, self.platform_agent_instance_id)
class TestDriverEgg(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def get_streamConfigs(self): raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') return raw_config, parsed_config ########################## # # The following tests generate different agent configs and pass them to a common base test script # ########################### @unittest.skip( "this test can't be run from coi services. it is missing dependencies") def test_driverLaunchModuleNoURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchNoModuleOnlyURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchBogusModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="bogus", driver_class="Bogus", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) @unittest.skip( "Launches an egg 'process' even though the egg download should produce error 404" ) def test_driverLaunchNoModule404URI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_404, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, False) def test_driverLaunchNoModuleBadEggURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_BAD, stream_configurations=[raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, True, False) def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True): """ This method runs a test of launching a driver with a given agent configuration """ # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) print 'new InstrumentModel id = %s ' % instModel_id # Create InstrumentAgent instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) print 'new InstrumentAgent id = %s' % instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new dp_id = %s' % data_product_id1 self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) print 'Data product streams1 = %s' % stream_ids # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id1 = %s' % dataset_ids[0] self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data self.get_datastore(self.parsed_dataset) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) print 'new dp_id = %s' % str(data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) print 'Data product streams2 = %s' % str(stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id2 = %s' % dataset_ids[0] self.raw_dataset = dataset_ids[0] # add start/stop for instrument agent gevent.joinall([ gevent.spawn( lambda: self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id)) ]) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) agent_process_id = ResourceAgentClient._get_agent_process_id( instDevice_id) print "Agent process id is '%s'" % str(agent_process_id) self.assertTrue(agent_process_id) gate = ProcessStateGate(self.processdispatchclient.read_process, agent_process_id, ProcessStateEnum.RUNNING) if not expect_launch: self.assertFalse( gate. await (30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" % agent_process_id) return self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % agent_process_id) print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING" #print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=agent_process_id, process=FakeProcess()) print "ResourceAgentClient created: %s" % str(self._ia_client) print "Sending command=ResourceAgentEvent.INITIALIZE" cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) if not expect_command: self.assertRaises(ServerError, self._ia_client.execute_agent, cmd) return retval = self._ia_client.execute_agent(cmd) print "Result of INITIALIZE: %s" % str(retval) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) print "Sending command=ResourceAgentEvent.RESET" cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) print "Result of RESET: %s" % str(reply)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() config.bootstrap.use_es = True self._start_container() self.addCleanup(TestActivateInstrumentIntegration.es_cleanup) self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() @staticmethod def es_cleanup(): es_host = CFG.get_safe('server.elasticsearch.host', 'localhost') es_port = CFG.get_safe('server.elasticsearch.port', '9200') es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10) indexes = STD_INDEXES.keys() indexes.append('%s_resources_index' % get_sys_name().lower()) indexes.append('%s_events_index' % get_sys_name().lower()) for index in indexes: IndexManagementService._es_call(es.river_couchdb_delete, index) IndexManagementService._es_call(es.index_delete, index) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name='', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name='notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification( notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification( notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id)) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument( self, expected_instrument_device_id='', extended_instrument=None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance( extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value)) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_instrument_device_id, notification.origin) self.assertEqual("instrument", notification.origin_type) self.assertEqual('ResourceLifecycleEvent', notification.event_type) def _check_computed_attributes_of_extended_product( self, expected_data_product_id='', extended_data_product=None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_data_product.computed.product_download_size_estimated, ComputedFloatValue) self.assertIsInstance( extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance( extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance( extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance( extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance( extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertIn( 'ok', extended_data_product.computed.last_granule.value['quality_flag']) self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value)) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_data_product_id, notification.origin) self.assertEqual("data product", notification.origin_type) self.assertEqual('DetectionEvent', notification.event_type) @attr('LOCOINT') #@unittest.skip('refactoring') @unittest.skipIf(not use_es, 'No ElasticSearch') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 90}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config, alerts=[]) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification(user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification(user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources( restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s", str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.INACTIVE, state) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.STOPPED, state) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print())) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] all_raw = "".join(raw_vals) # look for 't' entered after a prompt -- ">t" t_commands = all_raw.count(">t") if 10 != t_commands: log.error("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.error("raw val %s: %s", i, [r]) self.fail("Expected 10 't' strings in raw_vals, got %s" % t_commands) else: log.debug("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.debug("raw val %s: %s", i, [r]) replay_data_parsed = self.dataretrieverclient.retrieve( self.parsed_dataset) self.assertIsInstance(replay_data_parsed, Granule) rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt_parsed.pretty_print())) self.assertIn('temp', rdt_parsed) temp_vals = rdt_parsed['temp'] pressure_vals = rdt_parsed['pressure'] if 10 != len(temp_vals): log.error("%s temp_vals: %s", len(temp_vals), temp_vals) self.fail("Expected 10 temp_vals, got %s" % len(temp_vals)) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests. value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id1, extended_data_product=extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_1) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id2, extended_data_product=extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestCTDTransformsIntegration(IonIntegrationTestCase): pdict_id = None def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataset_management = self.datasetclient def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_instrument_model(self): instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_agent_instance(self, instAgent_id, instDevice_id): port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) return instAgentInstance_id def _create_param_dicts(self): tdom, sdom = time_series_domain() self.sdom = sdom.dump() self.tdom = tdom.dump() self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) def _create_input_data_products( self, ctd_stream_def_id, instDevice_id, ): dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=self.tdom, spatial_domain=self.sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product( dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_parsed_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) #--------------------------------------------------------------------------- # Create CTD Raw as the second data product #--------------------------------------------------------------------------- if not self.pdict_id: self._create_param_dicts() raw_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_RAW', parameter_dictionary_id=self.pdict_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=self.tdom, spatial_domain=self.sdom) ctd_raw_data_product = self.dataproductclient.create_data_product( dp_obj, raw_stream_def_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_raw_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids return ctd_parsed_data_product def _create_data_process_definitions(self): #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L1 Conductivity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L1 Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L1 Temperature: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L2 Salinity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #------------------------------------------------------------------------------------- # L2 Density: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) return self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id,\ self.ctd_L1_pressure_dprocdef_id,self.ctd_L1_temperature_dprocdef_id, \ self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id def _create_stream_definitions(self): if not self.pdict_id: self._create_param_dicts() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name='L0_Conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding='conductivity') outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name='L0_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding='pressure') outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name='L0_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding='temperature') return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id def _create_l0_output_data_products(self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id): out_data_prods = [] ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) out_data_prods.append(self.ctd_l0_conductivity_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_conductivity_output_dp_id) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) out_data_prods.append(self.ctd_l0_pressure_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_pressure_output_dp_id) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) out_data_prods.append(self.ctd_l0_temperature_output_dp_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l0_temperature_output_dp_id) return out_data_prods def _create_l1_out_data_products(self): ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0]) self.loggerpids.append(pid) ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0]) self.loggerpids.append(pid) ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0]) self.loggerpids.append(pid) def _create_l2_out_data_products(self): #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- if not self.pdict_id: self._create_param_dicts() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition( name='L2_salinity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding='salinity') outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition( name='L2_Density', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding='density') ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0]) self.loggerpids.append(pid) ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain=self.tdom, spatial_domain=self.sdom) self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) self.dataproductclient.activate_data_product_persistence( data_product_id=self.ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects( self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0]) self.loggerpids.append(pid) @unittest.skip('This test errors on coi-nightly, may be OBE.') def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance( instAgent_id, instDevice_id) #------------------------------------------------------------------------------------- # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------- self._create_param_dicts() ctd_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=self.pdict_id) #------------------------------------------------------------------------------------- # Create two data products #------------------------------------------------------------------------------------- ctd_parsed_data_product = self._create_input_data_products( ctd_stream_def_id, instDevice_id) #------------------------------------------------------------------------------------- # Create data process definitions #------------------------------------------------------------------------------------- self._create_data_process_definitions() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------------------------------------------------------------- outgoing_stream_l0_conductivity_id, \ outgoing_stream_l0_pressure_id, \ outgoing_stream_l0_temperature_id = self._create_stream_definitions() self.out_prod_ids = self._create_l0_output_data_products( outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id) self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition( name='L1_conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding='conductivity') self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition( name='L1_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding='pressure') self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition( name='L1_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding='temperature') self._create_l1_out_data_products() self._create_l2_out_data_products() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------------------------------------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=self.ctd_L0_all_dprocdef_id, in_data_product_ids=[ctd_parsed_data_product], out_data_product_ids=self.out_prod_ids) self.dataprocessclient.activate_data_process( ctd_l0_all_data_process_id) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) extended_process = self.dataprocessclient.get_data_process_extension( ctd_l0_all_data_process_id) self.assertEquals(extended_process.computed.operational_state.status, ComputedValueAvailability.NOTAVAILABLE) self.assertEquals(data_process.message_controllable, True) #------------------------------------------------------------------------------------- # L1 Conductivity: Create the data process #------------------------------------------------------------------------------------- l1_conductivity_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_conductivity_dprocdef_id, [self.ctd_l0_conductivity_output_dp_id], [self.ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process( l1_conductivity_data_process_id) data_process = self.rrclient.read(l1_conductivity_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l1_conductivity_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L1 Pressure: Create the data process #------------------------------------------------------------------------------------- l1_pressure_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_pressure_dprocdef_id, [self.ctd_l0_pressure_output_dp_id], [self.ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process( l1_pressure_data_process_id) data_process = self.rrclient.read(l1_pressure_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l1_pressure_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L1 Temperature: Create the data process #------------------------------------------------------------------------------------- l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L1_temperature_dprocdef_id, [self.ctd_l0_temperature_output_dp_id], [self.ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process( l1_temperature_all_data_process_id) data_process = self.rrclient.read(l1_temperature_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l1_temperature_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Salinity: Create the data process #------------------------------------------------------------------------------------- l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process( l2_salinity_all_data_process_id) data_process = self.rrclient.read(l2_salinity_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l2_salinity_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Density: Create the data process #------------------------------------------------------------------------------------- l2_density_all_data_process_id = self.dataprocessclient.create_data_process( self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_density_output_dp_id]) self.dataprocessclient.activate_data_process( l2_density_all_data_process_id) data_process = self.rrclient.read(l2_density_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=l2_density_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) # Wait for instrument agent to spawn gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Streaming #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #todo ResourceAgentClient no longer has method set_param # # Make sure the sampling rate and transmission are sane. # params = { # SBE37Parameter.NAVG : 1, # SBE37Parameter.INTERVAL : 5, # SBE37Parameter.TXREALTIME : True # } # self._ia_client.set_param(params) #todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it? cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) #todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it? # log.debug("test_activateInstrumentStream: calling go_observatory") # cmd = AgentCommand(command='go_observatory') # reply = self._ia_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #------------------------------------------------------------------------------------------------- # Cleanup processes #------------------------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) #-------------------------------------------------------------------------------- # Cleanup data products #-------------------------------------------------------------------------------- dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct, id_only=True) for dp_id in dp_ids: self.dataproductclient.delete_data_product(dp_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient( node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create( any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite)) platform_agent_instance_id, _ = self.RR.create( any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain=tdom, spatial_domain=sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension( instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Lifecycle self.assertEquals(len(extended_instrument.lcstate_transitions), 5) self.assertEquals( set(extended_instrument.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate'])) self.assertEquals(len(extended_instrument.availability_transitions), 2) self.assertEquals( set(extended_instrument.availability_transitions.keys()), set(['enable', 'announce'])) # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance( extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension( platform_device_id) self.assertEqual(1, len(extended_platform.portals)) self.assertEqual(1, len(extended_platform.portal_instruments)) #self.assertEqual(1, len(extended_platform.computed.portal_status.value)) # no agent started so NO statuses reported self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) self.assertEquals(extended_platform.platform_agent._id, platform_agent_id) self.assertEquals(len(extended_platform.lcstate_transitions), 5) self.assertEquals( set(extended_platform.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate'])) self.assertEquals(len(extended_platform.availability_transitions), 2) self.assertEquals( set(extended_platform.availability_transitions.keys()), set(['enable', 'announce'])) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual( ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue('Parsed_Canonical' in extended_instrument.computed. data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue('temp' in extended_instrument.computed. data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(DeviceStatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance( platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.OMS.force_delete_instrument_site(instrument_site_id) self.OMS.force_delete_platform_site(platform_site_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instModel_obj = IonObject(OT.CustomAttribute, name='SBE37IMModelAttr', description="model custom attr") instrument_model_id, _ = self.RR.create( any_old(RT.InstrumentModel, {"custom_attributes": [instModel_obj]})) instrument_device_id, _ = self.RR.create( any_old( RT.InstrumentDevice, { "custom_attributes": { "favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device( instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_data_producer(self): idevice_id = self.IMS.create_instrument_device( any_old(RT.InstrumentDevice)) self.assertEqual( 1, len( self.RR2. find_data_producer_ids_of_instrument_device_using_has_data_producer( idevice_id))) pdevice_id = self.IMS.create_platform_device(any_old( RT.PlatformDevice)) self.assertEqual( 1, len( self.RR2. find_data_producer_ids_of_platform_device_using_has_data_producer( pdevice_id))) @attr('PREP') def test_prepare_resource_support(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create( any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create( any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) instrument_device2_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_device3_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_device2_id, _ = self.RR.create(any_old(RT.PlatformDevice)) sensor_device2_id, _ = self.RR.create(any_old(RT.SensorDevice)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device2_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_device2_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device2_id, PRED.hasDevice, instrument_device2_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(sensor_device2_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device2_id, PRED.hasDevice, instrument_device2_id) sensor_model_id #is only a target #set lcstate - used for testing prepare - not setting all to DEVELOP, only some self.RR.execute_lifecycle_transition(instrument_agent_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_agent_id, LCE.DEVELOP) #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain=tdom, spatial_domain=sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner( instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") def ion_object_encoder(obj): return obj.__dict__ #First call to create instrument_data = self.IMS.prepare_instrument_device_support() #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, '') self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual( len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual( instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. resources), 1) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. resources[0]._id, instrument_agent_instance_id) self.assertEqual( len(instrument_data.associations['InstrumentModel']. associated_resources), 0) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. associated_resources), 0) self.assertEqual( len(instrument_data.associations['SensorDevice'].resources), 0) #Next call to update instrument_data = self.IMS.prepare_instrument_device_support( instrument_device_id) #print 'Update results' #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, instrument_device_id) self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual( len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual( instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. resources), 1) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. resources[0]._id, instrument_agent_instance_id) self.assertEqual( len(instrument_data.associations['InstrumentModel']. associated_resources), 1) self.assertEqual( instrument_data.associations['InstrumentModel']. associated_resources[0].s, instrument_device_id) self.assertEqual( instrument_data.associations['InstrumentModel']. associated_resources[0].o, instrument_model_id) self.assertEqual( len(instrument_data.associations['InstrumentAgentInstance']. associated_resources), 1) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. associated_resources[0].o, instrument_agent_instance_id) self.assertEqual( instrument_data.associations['InstrumentAgentInstance']. associated_resources[0].s, instrument_device_id) self.assertEqual( len(instrument_data.associations['SensorDevice'].resources), 1) self.assertEqual( instrument_data.associations['SensorDevice'].resources[0]._id, sensor_device_id) self.assertEqual( len(instrument_data.associations['SensorDevice']. associated_resources), 1) self.assertEqual( instrument_data.associations['SensorDevice']. associated_resources[0].o, instrument_device_id) self.assertEqual( instrument_data.associations['SensorDevice']. associated_resources[0].s, sensor_device_id) self.assertEqual( instrument_data.associations['InstrumentModel'].assign_request. request_parameters['instrument_device_id'], instrument_device_id) #test prepare for create of instrument agent instance instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support( ) #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, '') self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. resources), 2) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent'].resources ), 1) self.assertEqual( instrument_agent_data.associations['InstrumentAgent'].resources[0]. _id, instrument_agent_id) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. associated_resources), 0) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent']. associated_resources), 0) #test prepare for update of instrument agent instance to see if it is associated with the instrument that was created instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support( instrument_agent_instance_id=instrument_agent_instance_id) #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, instrument_agent_instance_id) self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. resources), 3) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent'].resources ), 1) self.assertEqual( instrument_agent_data.associations['InstrumentAgent'].resources[0]. _id, instrument_agent_id) self.assertEqual( len(instrument_agent_data.associations['InstrumentDevice']. associated_resources), 1) self.assertEqual( instrument_agent_data.associations['InstrumentDevice']. associated_resources[0].s, instrument_device_id) self.assertEqual( instrument_agent_data.associations['InstrumentDevice']. associated_resources[0].o, instrument_agent_instance_id) self.assertEqual( len(instrument_agent_data.associations['InstrumentAgent']. associated_resources), 1) self.assertEqual( instrument_agent_data.associations['InstrumentAgent']. associated_resources[0].o, instrument_agent_id) self.assertEqual( instrument_agent_data.associations['InstrumentAgent']. associated_resources[0].s, instrument_agent_instance_id) self.assertEqual( instrument_agent_data.associations['InstrumentAgent']. assign_request.request_parameters['instrument_agent_instance_id'], instrument_agent_instance_id) #test prepare for update of data product to see if it is associated with the instrument that was created data_product_data = self.DP.prepare_data_product_support( data_product_id1) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent=2) self.assertEqual(data_product_data._id, data_product_id1) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual( len(data_product_data.associations['StreamDefinition'].resources), 1) self.assertEqual( len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual( len(data_product_data.associations['StreamDefinition']. associated_resources), 1) self.assertEqual( data_product_data.associations['StreamDefinition']. associated_resources[0].s, data_product_id1) self.assertEqual( len(data_product_data.associations['Dataset'].associated_resources ), 0) self.assertEqual( len(data_product_data. associations['InstrumentDeviceHasOutputProduct'].resources), 3) self.assertEqual( len(data_product_data.associations[ 'InstrumentDeviceHasOutputProduct'].associated_resources), 1) self.assertEqual( data_product_data.associations['InstrumentDeviceHasOutputProduct']. associated_resources[0].s, instrument_device_id) self.assertEqual( data_product_data.associations['InstrumentDeviceHasOutputProduct']. associated_resources[0].o, data_product_id1) self.assertEqual( len(data_product_data.associations['PlatformDevice'].resources), 2) platform_data = self.IMS.prepare_platform_device_support() #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, '') self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual( len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual( platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual( len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual( platform_data.associations['PlatformAgentInstance'].resources[0]. _id, platform_agent_instance_id) self.assertEqual( len(platform_data.associations['PlatformModel']. associated_resources), 0) self.assertEqual( len(platform_data.associations['PlatformAgentInstance']. associated_resources), 0) self.assertEqual( len(platform_data.associations['InstrumentDevice'].resources), 1) platform_data = self.IMS.prepare_platform_device_support( platform_device_id) #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, platform_device_id) self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual( len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual( platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual( len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual( platform_data.associations['PlatformAgentInstance'].resources[0]. _id, platform_agent_instance_id) self.assertEqual( len(platform_data.associations['PlatformModel']. associated_resources), 1) self.assertEqual( platform_data.associations['PlatformModel']. associated_resources[0].s, platform_device_id) self.assertEqual( platform_data.associations['PlatformModel']. associated_resources[0].o, platform_model_id) self.assertEqual( len(platform_data.associations['PlatformAgentInstance']. associated_resources), 1) self.assertEqual( platform_data.associations['PlatformAgentInstance']. associated_resources[0].o, platform_agent_instance_id) self.assertEqual( platform_data.associations['PlatformAgentInstance']. associated_resources[0].s, platform_device_id) self.assertEqual( len(platform_data.associations['InstrumentDevice'].resources), 2) #self.assertEqual(len(platform_data.associations['InstrumentDevice'].associated_resources), 1) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].s, platform_device_id) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_device_id) self.assertEqual( platform_data.associations['PlatformModel'].assign_request. request_parameters['platform_device_id'], platform_device_id) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.RR2.pluck(sensor_device2_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_device(instrument_device2_id) self.IMS.force_delete_platform_agent_instance( platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_device(platform_device2_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_device(sensor_device2_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id)
class TestDataProcessWithLookupTable(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.dataset_management = self.datasetclient self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) def test_lookupTableProcessing(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentModel id = %s', instModel_id) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD) try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice and attachment for lookup table #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") try: instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) log.info( 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s', instDevice_id) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(instDevice_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance") self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) log.info( 'TestDataProcessWithLookupTable: new Stream Definition id = %s', instDevice_id) log.info('Creating new CDM data product with a stream definition') # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product( dp_obj, ctd_stream_def_id) log.info('new ctd_parsed_data_product_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.info('TestDataProcessWithLookupTable: Data product streams1 = %s', stream_ids) #------------------------------- # Create CTD Raw as the second data product #------------------------------- log.info( 'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition' ) raw_stream_def_id = self.pubsubclient.create_stream_definition( name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj = IonObject(RT.DataProduct, name='ctd_raw', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) ctd_raw_data_product = self.dataproductclient.create_data_product( dp_obj, raw_stream_def_id) log.info('new ctd_raw_data_product_id = %s', ctd_raw_data_product) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.info('Data product streams2 = %s', stream_ids) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug( "TestDataProcessWithLookupTable: create data process definition ctd_L0_all" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new ctd_L0_all data process definition: %s" % ex) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..." att = IonObject(RT.Attachment, name='processDefinitionLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processDefinitionAttachment = self.rrclient.create_attachment( ctd_L0_all_dprocdef_id, att) log.debug( "TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment)) processDefinitionAttachment_obj = self.rrclient.read( processDefinitionAttachment) log.debug( "TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj)) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity') outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure') outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature') log.debug( "TestDataProcessWithLookupTable: create output data product L0 conductivity" ) ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug( "TestDataProcessWithLookupTable: create output data product L0 pressure" ) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug( "TestDataProcessWithLookupTable: create output data product L0 temperature" ) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain=tdom, spatial_domain=sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug( "TestDataProcessWithLookupTable: create L0 all data_process start") try: in_prods = [] in_prods.append(ctd_parsed_data_product) out_prods = [ ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id ] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id=ctd_L0_all_dprocdef_id, in_data_product_ids=in_prods, out_data_product_ids=out_prods) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "TestDataProcessWithLookupTable: create L0 all data_process return" ) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects( subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, object_type=RT.Process, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..." att = IonObject(RT.Attachment, name='processLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) processAttachment = self.rrclient.create_attachment( ctd_l0_all_data_process_id, att) log.info( 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s', processAttachment)
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateRSNVel3DInstrument, self).setUp() config = DotDict() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid @attr('LOCOINT') @unittest.skip('under construction') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 180}}}) def test_activate_rsn_vel3d(self): log.info("--------------------------------------------------------------------------------------------------------") # load_parameter_scenarios self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict( op="load", scenario="BETA", path="master", categories="ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition", clearcols="owner_id,org_ids", assets="res/preload/r2_ioc/ooi_assets", parseooi="True", )) self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='Vel3DMModel', description="Vel3DMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'test_activate_rsn_vel3d new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw' ) vel3d_b_sample = StreamConfiguration(stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample') vel3d_b_engineering = StreamConfiguration(stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering') RSN_VEL3D_01 = { 'DEV_ADDR' : "10.180.80.6", 'DEV_PORT' : 2101, 'DATA_PORT' : 1026, 'CMD_PORT' : 1025, 'PA_BINARY' : "port_agent" } # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='Vel3DAgent', description="Vel3DAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg", stream_configurations = [raw_config, vel3d_b_sample, vel3d_b_engineering]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D ') instDevice_obj = IonObject(RT.InstrumentDevice, name='Vel3DDevice', description="Vel3DDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s " , instDevice_id) port_agent_config = { 'device_addr': '10.180.80.6', 'device_port': 2101, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 1025, 'data_port': 1026, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='Vel3DAgentInstance', description="Vel3DAgentInstance", port_agent_config = port_agent_config, alerts= []) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_sample', id_only=True) parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_sample', parameter_dictionary_id=parsed_sample_pdict_id) parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_engineering', id_only=True) parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_engineering', parameter_dictionary_id=parsed_eng_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='vel3d_b_sample', description='vel3d_b_sample', temporal_domain = tdom, spatial_domain = sdom) sample_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_sample_stream_def_id) log.debug( 'new dp_id = %s' , sample_data_product_id) self.dpclient.activate_data_product_persistence(data_product_id=sample_data_product_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=sample_data_product_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasStream, None, True) log.debug('sample_data_product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for sample_data_product = %s' , dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('vel3d_b_sample', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='vel3d_b_engineering', description='vel3d_b_engineering', temporal_domain = tdom, spatial_domain = sdom) eng_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id) log.debug( 'new dp_id = %s' , eng_data_product_id) self.dpclient.activate_data_product_persistence(data_product_id=eng_data_product_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=eng_data_product_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('test_activate_rsn_vel3d Data product streams2 = %s' , str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s' , dataset_ids[0]) self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) def check_state(label, desired_state): actual_state = self._ia_client.get_agent_state() log.debug("%s instrument agent is in state '%s'", label, actual_state) self.assertEqual(desired_state, actual_state) log.debug("test_activate_rsn_vel3d: got ia client %s" , str(self._ia_client)) check_state("just-spawned", ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: initialize %s" , str(retval)) check_state("initialized", ResourceAgentState.INACTIVE) log.debug("test_activate_rsn_vel3d Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return value from go_active %s" , str(reply)) check_state("activated", ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending go_active command %s" , str(state)) # cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) check_state("commanded", ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("current state after sending run command %s" , str(state)) # cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # gevent.sleep(5) # # cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE) # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activate_rsn_vel3d: run %s" , str(reply)) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state)) # # cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.STOPPED, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RESUME) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) # # cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.IDLE, state) # # cmd = AgentCommand(command=ResourceAgentEvent.RUN) # retval = self._ia_client.execute_agent(cmd) # state = self._ia_client.get_agent_state() # self.assertEqual(ResourceAgentState.COMMAND, state) log.debug( "test_activate_rsn_vel3d: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activate_rsn_vel3d: return from reset %s" , str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) ) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(sample_data_product_id) self.dpclient.delete_data_product(eng_data_product_id) self.dpclient.delete_data_product(data_product_id2)
class VisStreamLauncher(ImmediateProcess): """ Class emulates a stream source from a NetCDF file. It emits a record of data every few seconds on a stream identified by a routing key. """ def on_init(self): log.debug("VizStreamProducer init. Self.id=%s" % self.id) def on_start(self): log.debug("VizStreamProducer start") self.data_source_name = self.CFG.get('name') self.dataset = self.CFG.get('dataset') # create a pubsub client and a resource registry client self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) # Dummy instrument related clients self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.IngestClient = IngestionManagementServiceClient( node=self.container.node) # create the pubsub client self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) # Additional code for creating a dummy instrument """ # Set up the preconditions. Look for an existing ingestion config while True: log.info("VisStreamLauncher:on_start: Waiting for an ingestion configuration to be available.") ingestion_cfgs, _ = self.rrclient.find_resources(RT.IngestionConfiguration, None, None, True) if len(ingestion_cfgs) > 0: break else: gevent.sleep(1) """ # Check to see if the data_product already exists in the system (for e.g re launching the code after a crash) dp_ids, _ = self.rrclient.find_resources(RT.DataProduct, None, self.data_source_name, True) if len(dp_ids) > 0: data_product_id = dp_ids[0] print '>>>>>>>>>>>>> Found dp_id = ', data_product_id else: # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name=self.data_source_name, description=self.data_source_name, model_label=self.data_source_name) instModel_id = self.imsclient.create_instrument_model( instModel_obj) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name=self.data_source_name, description=self.data_source_name, serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition( container=ctd_stream_def) print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct, name=self.data_source_name, description='ctd stream test') data_product_id = self.dpclient.create_data_product( dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=data_product_id) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id, persist_data=True, persist_metadata=True) print '>>>>>>>>>>>> New dp_id = ', data_product_id # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id, PRED.hasStream, None, True) if self.dataset == 'sinusoidal': pid = self.container.spawn_process( name='ctd_test.' + self.data_source_name, module='ion.processes.data.sinusoidal_stream_publisher', cls='SinusoidalCtdPublisher', config={'process': { 'stream_id': stream_ids[0] }}) else: pid = self.container.spawn_process( name='ctd_test.' + self.data_source_name, module='ion.processes.data.ctd_stream_publisher', cls='SimpleCtdPublisher', config={'process': { 'stream_id': stream_ids[0] }}) def on_quit(self): log.debug("VizStreamProducer quit")
class TestInstrumentAlerts(IonIntegrationTestCase): pdict_id = None def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.catch_alert= gevent.queue.Queue() def _create_instrument_model(self): instModel_obj = IonObject( RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) self.addCleanup(self.imsclient.delete_instrument_model, instModel_id) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.addCleanup(self.imsclient.delete_instrument_agent, instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) self.addCleanup(self.imsclient.delete_instrument_device, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_stream_alarms(self, instDevice_id): #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ temp_alert_def1 = { 'name' : 'temperature_warning_interval temp below 25', 'stream_name' : 'parsed', 'description' : 'temperature_warning_interval temp below 25', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'resource_id' : instDevice_id, 'origin_type' : 'device', #'lower_bound' : 0, #'lower_rel_op' : '<', # temp 'upper_rel_op' : '<=', 'upper_bound' : 25, 'alert_class' : 'IntervalAlert' } temp_alert_def2 = { 'name' : 'temperature_warning_interval temp below 50', 'stream_name' : 'parsed', 'description' : 'temperature_warning_interval temp below 50', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'resource_id' : instDevice_id, 'origin_type' : 'device', #'lower_bound' : 25, #'lower_rel_op' : '<', # temp 'upper_rel_op' : '<=', 'upper_bound' : 50, 'alert_class' : 'IntervalAlert' } temp_alert_def3 = { 'name' : 'temperature_warning_interval temp below 75', 'stream_name' : 'parsed', 'description' : 'temperature_warning_interval temp below 75', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'resource_id' : instDevice_id, 'origin_type' : 'device', #'lower_bound' : 50, #'lower_rel_op' : '<', # temp 'upper_rel_op' : '<=', 'upper_bound' : 75, 'alert_class' : 'IntervalAlert' } late_data_alert_def = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'description' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'value_id' : None, 'resource_id' : instDevice_id, 'origin_type' : 'device', 'time_delta' : 2, 'alert_class' : 'LateDataAlert' } return [temp_alert_def1, temp_alert_def2, temp_alert_def3, late_data_alert_def] def _create_instrument_agent_instance(self, instAgent_id, instDevice_id): port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } self.all_alerts = self._create_instrument_stream_alarms(instDevice_id) instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config, alerts= self.all_alerts ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) self.addCleanup(self.imsclient.delete_instrument_agent_instance, instAgentInstance_id) return instAgentInstance_id def test_alerts(self): # # test that with the 4009 sim we can get a late data alert # as well as alerts for out of range for > 25, > 50, and > 75 # as well as the ALL_CLEAR alerts for each of them. # #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) # It is necessary for the instrument device to be associated with atleast one output data product parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) # We are creating two data products here, one for parsed and another raw dp_obj_parsed = IonObject(RT.DataProduct, name='parsed_data_product', description='Parsed output data product for instrument') dp_obj_raw = IonObject(RT.DataProduct, name='raw_data_prod', description='Raw output data product for instrument') parsed_out_data_prod_id = self.dataproductclient.create_data_product(data_product=dp_obj_parsed, stream_definition_id=parsed_stream_def_id) raw_out_data_prod_id = self.dataproductclient.create_data_product(data_product=dp_obj_raw, stream_definition_id=raw_stream_def_id) self.addCleanup(self.dataproductclient.delete_data_product, parsed_out_data_prod_id) self.addCleanup(self.dataproductclient.delete_data_product, raw_out_data_prod_id) self.dataproductclient.activate_data_product_persistence(data_product_id=parsed_out_data_prod_id) self.dataproductclient.activate_data_product_persistence(data_product_id=raw_out_data_prod_id) # todo: note that the generated config on the instruments will be done for both raw and parsed stream defs since these two data products constructed with each are associated as output data products with the instrument # todo: if the config is not generated for a stream def, then the instrument agent will complain if the simulator generates data corresponding to a stream def that is not there in the stream config as a mentioned stream def self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=parsed_out_data_prod_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=raw_out_data_prod_id) log.debug("assigned instdevice id: %s to data product: %s", instDevice_id, raw_out_data_prod_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id,instDevice_id ) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) # Wait for instrument agent to spawn gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Set up the subscriber to catch the alert event #------------------------------------------------------------------------------------- def callback_for_alert(event, *args, **kwargs): log.debug("caught an alert: %s", event) self.catch_alert.put(event) self.event_subscriber = EventSubscriber(event_type='DeviceStatusAlertEvent', origin=instDevice_id, callback=callback_for_alert) self.event_subscriber.start() self.addCleanup(self.event_subscriber.stop) #------------------------------------------------------------------------------------- # Running the instrument.... #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) # Prevent this test from hanging indefinitely until # OOIION-1313 is resolved timeout_val = 90 with gevent.Timeout(timeout_val, Exception('Agent failed to initialize after %fs' % timeout_val)): reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) with gevent.Timeout(timeout_val, Exception('Agent failed to go active after %fs' % timeout_val)): reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) with gevent.Timeout(timeout_val, Exception('Agent failed to get resource after %fs' % timeout_val)): retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) with gevent.Timeout(timeout_val, Exception('Agent failed to run after %fs' % timeout_val)): reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) with gevent.Timeout(timeout_val, Exception('Agent failed to start autosample after %fs' % timeout_val)): retval = self._ia_client.execute_resource(cmd) got_bad_temp = [False, False, False, False] got_late_data = False got_temp_all_clear = [False, False, False, False] runtime = 0 starttime = time.time() caught_events = [] while (got_bad_temp[0] == False or got_bad_temp[1] == False or got_bad_temp[2] == False or got_temp_all_clear[0] == False or got_temp_all_clear[1] == False or got_temp_all_clear[2] == False or got_late_data == False) and runtime < 120: a = self.catch_alert.get(timeout=180) caught_events.append(a) if a.name == 'temperature_warning_interval temp below 25': if a.sub_type == 'WARNING' and \ a.values[0] > 25: got_bad_temp[0] = True log.error(str(a.values[0]) + " should be above 25") elif a.sub_type == 'ALL_CLEAR': got_temp_all_clear[0] = True log.debug("25 ALL_CLEAR") if a.name == 'temperature_warning_interval temp below 50': if a.sub_type == 'WARNING' and \ a.values[0] > 50: got_bad_temp[1] = True log.error(str(a.values[0]) + " should be above 50") elif a.sub_type == 'ALL_CLEAR': got_temp_all_clear[1] = True log.debug("50 ALL_CLEAR" ) if a.name == 'temperature_warning_interval temp below 75': if a.sub_type == 'WARNING' and \ a.values[0] > 75: got_bad_temp[2] = True log.error(str(a.values[0]) + " should be above 75") elif a.sub_type == 'ALL_CLEAR': got_temp_all_clear[2] = True log.debug("75 ALL_CLEAR") if a.name == 'late_data_warning' and \ a.description == 'Expected data has not arrived.': got_late_data = True log.debug("late value") runtime = time.time() - starttime log.debug("caught_events: %s", [c.name for c in caught_events]) for c in caught_events: self.assertIn(c.name, ['temperature_warning_interval temp below 25', 'temperature_warning_interval temp below 50', 'temperature_warning_interval temp below 75', 'late_data_warning']) self.assertEqual(c.origin, instDevice_id) self.assertEqual(c.type_, 'DeviceStatusAlertEvent') self.assertEqual(c.origin_type, 'InstrumentDevice') self.assertTrue(got_bad_temp) #simply log the state of the late data flag and the simulator does not seem to be consistent. log.debug("test_alerts late data alert flag: %s", got_late_data)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' unittest # suppress an pycharm inspector error if all unittest.skip references are commented out self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode as it depends on modifying CFG on service side') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ self.patch_cfg(CFG["container"], {"extended_resources": {"strip_results": False}}) #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_site_id, _ = self.RR.create(any_old(RT.InstrumentSite)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_site_id, _ = self.RR.create(any_old(RT.PlatformSite)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical') pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) # Lifecycle self.assertEquals(len(extended_instrument.lcstate_transitions), 6) self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate', 'delete'])) self.assertEquals(len(extended_instrument.availability_transitions), 2) self.assertEquals(set(extended_instrument.availability_transitions.keys()), set(['enable', 'announce'])) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) log.debug("extended_instrument.computed: %s", extended_instrument.computed) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) extended_platform = self.IMS.get_platform_device_extension(platform_device_id) self.assertEqual(1, len(extended_platform.portals)) self.assertEqual(1, len(extended_platform.portal_instruments)) #self.assertEqual(1, len(extended_platform.computed.portal_status.value)) # no agent started so NO statuses reported self.assertEqual(1, len(extended_platform.instrument_devices)) self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id) self.assertEqual(1, len(extended_platform.instrument_models)) self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id) self.assertEquals(extended_platform.platform_agent._id, platform_agent_id) self.assertEquals(len(extended_platform.lcstate_transitions), 6) self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['develop', 'deploy', 'retire', 'plan', 'integrate', 'delete'])) self.assertEquals(len(extended_platform.availability_transitions), 2) self.assertEquals(set(extended_platform.availability_transitions.keys()), set(['enable', 'announce'])) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) ##check data_product_parameters_set # !!! OOIION-1342 The UI does not use data_product_parameters_set and it is an expensive calc so the attribute calc was disabled # !!! Remove check in this test #self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_product_parameters_set.status) #self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) ## the ctd parameters should include 'temp' #self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.firmware_version.status) # self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, # extended_instrument.computed.operational_state.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.power_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.communications_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.data_status_roll_up.status) # self.assertEqual(DeviceStatusType.STATUS_OK, # extended_instrument.computed.data_status_roll_up.value) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.OMS.force_delete_instrument_site(instrument_site_id) self.OMS.force_delete_platform_site(platform_site_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instModel_obj = IonObject(OT.CustomAttribute, name='SBE37IMModelAttr', description="model custom attr") instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": [instModel_obj] })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_data_producer(self): idevice_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice)) self.assertEqual(1, len(self.RR2.find_data_producer_ids_of_instrument_device_using_has_data_producer(idevice_id))) pdevice_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) self.assertEqual(1, len(self.RR2.find_data_producer_ids_of_platform_device_using_has_data_producer(pdevice_id))) @attr('PREP') def test_prepare_resource_support(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) instrument_device2_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) instrument_device3_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_device2_id, _ = self.RR.create(any_old(RT.PlatformDevice)) sensor_device2_id, _ = self.RR.create(any_old(RT.SensorDevice)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device2_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(platform_device2_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device2_id, PRED.hasDevice, instrument_device2_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(sensor_device2_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device2_id, PRED.hasDevice, instrument_device2_id) sensor_model_id #is only a target #set lcstate - used for testing prepare - not setting all to DEVELOP, only some self.RR.execute_lifecycle_transition(instrument_agent_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(instrument_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_device2_id, LCE.DEVELOP) self.RR.execute_lifecycle_transition(platform_agent_id, LCE.DEVELOP) #create a parsed product for this instrument output dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical') pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") def ion_object_encoder(obj): return obj.__dict__ #First call to create instrument_data = self.IMS.prepare_instrument_device_support() #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, '') self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual(len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual(instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].resources), 1) self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].resources[0]._id, instrument_agent_instance_id) self.assertEqual(len(instrument_data.associations['InstrumentModel'].associated_resources), 0) self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].associated_resources), 0) self.assertEqual(len(instrument_data.associations['SensorDevice'].resources), 0) #Next call to update instrument_data = self.IMS.prepare_instrument_device_support(instrument_device_id) #print 'Update results' #print simplejson.dumps(instrument_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_data._id, instrument_device_id) self.assertEqual(instrument_data.type_, OT.InstrumentDevicePrepareSupport) self.assertEqual(len(instrument_data.associations['InstrumentModel'].resources), 1) self.assertEqual(instrument_data.associations['InstrumentModel'].resources[0]._id, instrument_model_id) self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].resources), 1) self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].resources[0]._id, instrument_agent_instance_id) self.assertEqual(len(instrument_data.associations['InstrumentModel'].associated_resources), 1) self.assertEqual(instrument_data.associations['InstrumentModel'].associated_resources[0].s, instrument_device_id) self.assertEqual(instrument_data.associations['InstrumentModel'].associated_resources[0].o, instrument_model_id) self.assertEqual(len(instrument_data.associations['InstrumentAgentInstance'].associated_resources), 1) self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].associated_resources[0].o, instrument_agent_instance_id) self.assertEqual(instrument_data.associations['InstrumentAgentInstance'].associated_resources[0].s, instrument_device_id) self.assertEqual(len(instrument_data.associations['SensorDevice'].resources), 1) self.assertEqual(instrument_data.associations['SensorDevice'].resources[0]._id, sensor_device_id) self.assertEqual(len(instrument_data.associations['SensorDevice'].associated_resources), 1) self.assertEqual(instrument_data.associations['SensorDevice'].associated_resources[0].o, instrument_device_id) self.assertEqual(instrument_data.associations['SensorDevice'].associated_resources[0].s, sensor_device_id) self.assertEqual(instrument_data.associations['InstrumentModel'].assign_request.request_parameters['instrument_device_id'], instrument_device_id) #test prepare for create of instrument agent instance instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support() #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, '') self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].resources), 2) self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].resources), 1) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].resources[0]._id, instrument_agent_id) self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].associated_resources), 0) self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].associated_resources), 0) #test prepare for update of instrument agent instance to see if it is associated with the instrument that was created instrument_agent_data = self.IMS.prepare_instrument_agent_instance_support(instrument_agent_instance_id=instrument_agent_instance_id) #print 'Update results' #print simplejson.dumps(instrument_agent_data, default=ion_object_encoder, indent=2) self.assertEqual(instrument_agent_data._id, instrument_agent_instance_id) self.assertEqual(instrument_agent_data.type_, OT.InstrumentAgentInstancePrepareSupport) self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].resources), 3) self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].resources), 1) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].resources[0]._id, instrument_agent_id) self.assertEqual(len(instrument_agent_data.associations['InstrumentDevice'].associated_resources), 1) self.assertEqual(instrument_agent_data.associations['InstrumentDevice'].associated_resources[0].s, instrument_device_id) self.assertEqual(instrument_agent_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_agent_instance_id) self.assertEqual(len(instrument_agent_data.associations['InstrumentAgent'].associated_resources), 1) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].associated_resources[0].o, instrument_agent_id) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].associated_resources[0].s, instrument_agent_instance_id) self.assertEqual(instrument_agent_data.associations['InstrumentAgent'].assign_request.request_parameters['instrument_agent_instance_id'], instrument_agent_instance_id) #test prepare for update of data product to see if it is associated with the instrument that was created data_product_data = self.DP.prepare_data_product_support(data_product_id1) #print simplejson.dumps(data_product_data, default=ion_object_encoder, indent=2) self.assertEqual(data_product_data._id, data_product_id1) self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport) self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 1) self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0) self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1) self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, data_product_id1) self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0) self.assertEqual(len(data_product_data.associations['InstrumentDeviceHasOutputProduct'].resources), 3) self.assertEqual(len(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources), 1) self.assertEqual(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources[0].s, instrument_device_id) self.assertEqual(data_product_data.associations['InstrumentDeviceHasOutputProduct'].associated_resources[0].o, data_product_id1) self.assertEqual(len(data_product_data.associations['PlatformDevice'].resources), 2) platform_data = self.IMS.prepare_platform_device_support() #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, '') self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual(len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual(platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual(platform_data.associations['PlatformAgentInstance'].resources[0]._id, platform_agent_instance_id) self.assertEqual(len(platform_data.associations['PlatformModel'].associated_resources), 0) self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].associated_resources), 0) self.assertEqual(len(platform_data.associations['InstrumentDevice'].resources), 1) platform_data = self.IMS.prepare_platform_device_support(platform_device_id) #print simplejson.dumps(platform_data, default=ion_object_encoder, indent=2) self.assertEqual(platform_data._id, platform_device_id) self.assertEqual(platform_data.type_, OT.PlatformDevicePrepareSupport) self.assertEqual(len(platform_data.associations['PlatformModel'].resources), 1) self.assertEqual(platform_data.associations['PlatformModel'].resources[0]._id, platform_model_id) self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].resources), 1) self.assertEqual(platform_data.associations['PlatformAgentInstance'].resources[0]._id, platform_agent_instance_id) self.assertEqual(len(platform_data.associations['PlatformModel'].associated_resources), 1) self.assertEqual(platform_data.associations['PlatformModel'].associated_resources[0].s, platform_device_id) self.assertEqual(platform_data.associations['PlatformModel'].associated_resources[0].o, platform_model_id) self.assertEqual(len(platform_data.associations['PlatformAgentInstance'].associated_resources), 1) self.assertEqual(platform_data.associations['PlatformAgentInstance'].associated_resources[0].o, platform_agent_instance_id) self.assertEqual(platform_data.associations['PlatformAgentInstance'].associated_resources[0].s, platform_device_id) self.assertEqual(len(platform_data.associations['InstrumentDevice'].resources), 2) #self.assertEqual(len(platform_data.associations['InstrumentDevice'].associated_resources), 1) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].s, platform_device_id) #self.assertEqual(platform_data.associations['InstrumentDevice'].associated_resources[0].o, instrument_device_id) self.assertEqual(platform_data.associations['PlatformModel'].assign_request.request_parameters['platform_device_id'], platform_device_id) # cleanup c = DotDict() c.resource_registry = self.RR self.RR2.pluck(instrument_agent_id) self.RR2.pluck(instrument_model_id) self.RR2.pluck(instrument_device_id) self.RR2.pluck(platform_agent_id) self.RR2.pluck(sensor_device_id) self.RR2.pluck(sensor_device2_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_device(instrument_device2_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_device(platform_device2_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_device(sensor_device2_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dmpsclient = DataProductManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.psmsclient = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_platform_site(site_id, deployment_id) self.imsclient.deploy_platform_device(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) ) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.RR2.pluck(deployment_id) self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self): #------------------------------------------------------------------------------------- # Create platform site, platform device, platform model #------------------------------------------------------------------------------------- platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') platform_site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device(platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model(platform_model__obj) #------------------------------------------------------------------------------------- # Create instrument site #------------------------------------------------------------------------------------- instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id) #---------------------------------------------------------------------------------------------------- # Create an instrument device #---------------------------------------------------------------------------------------------------- instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) #---------------------------------------------------------------------------------------------------- # Create an instrument model #---------------------------------------------------------------------------------------------------- instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj) #---------------------------------------------------------------------------------------------------- # Create a deployment object #---------------------------------------------------------------------------------------------------- start = IonTime(datetime.datetime(2013,1,1)) end = IonTime(datetime.datetime(2014,1,1)) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string()) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', context=IonObject(OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_platform_model_to_platform_device(res.platform_model_id, res.platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_platform_model_to_platform_site(res.platform_model_id, res.platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("adding instrument site and device to deployment") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("adding platform site and device to deployment") self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id) self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) log.debug("deactivatin deployment, expecting success") self.omsclient.deactivate_deployment(res.deployment_id) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without site+device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id) log.debug("activating deployment without instrument site, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "Devices in this deployment outnumber sites") #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id) log.debug("activating deployment without device, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment") def test_activate_deployment_asymmetric_children(self): """ P0 | \ P1 P2 | I1 Complex deployment using CSP P1, P2, and P3 share the same platform model. The CSP solver should be able to work this out based on relationships to parents """ log.debug("create models") imodel_id = self.RR2.create(any_old(RT.InstrumentModel)) pmodel_id = self.RR2.create(any_old(RT.PlatformModel)) log.debug("create devices") idevice_id = self.RR2.create(any_old(RT.InstrumentDevice)) pdevice_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(3)] log.debug("create sites") isite_id = self.RR2.create(any_old(RT.InstrumentSite)) psite_id = [self.RR2.create(any_old(RT.PlatformSite)) for _ in range(3)] log.debug("assign models") self.RR2.assign_instrument_model_to_instrument_device_with_has_model(imodel_id, idevice_id) self.RR2.assign_instrument_model_to_instrument_site_with_has_model(imodel_id, isite_id) for x in range(3): self.RR2.assign_platform_model_to_platform_device_with_has_model(pmodel_id, pdevice_id[x]) self.RR2.assign_platform_model_to_platform_site_with_has_model(pmodel_id, psite_id[x]) log.debug("assign hierarchy") self.RR2.assign_instrument_device_to_platform_device_with_has_device(idevice_id, pdevice_id[1]) self.RR2.assign_instrument_site_to_platform_site_with_has_site(isite_id, psite_id[1]) for x in range(1,3): self.RR2.assign_platform_device_to_platform_device_with_has_device(pdevice_id[x], pdevice_id[0]) self.RR2.assign_platform_site_to_platform_site_with_has_site(psite_id[x], psite_id[0]) log.debug("create and activate deployment") dep_id = self.RR2.create(any_old(RT.Deployment, {"context": IonObject(OT.RemotePlatformDeploymentContext)})) self.RR2.assign_deployment_to_platform_device_with_has_deployment(dep_id, pdevice_id[0]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(dep_id, psite_id[0]) self.omsclient.activate_deployment(dep_id) log.debug("verifying deployment") self.assertEqual(idevice_id, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(isite_id), "The instrument device was not assigned to the instrument site") for x in range(3): self.assertEqual(pdevice_id[x], self.RR2.find_platform_device_id_of_platform_site_using_has_device(psite_id[x]), "Platform device %d was not assigned to platform site %d" % (x, x)) def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message="did not specify fail_message"): with self.assertRaises(err_type) as cm: self.omsclient.activate_deployment(deployment_id) self.assertIn(fail_message, cm.exception.message) def test_3x3_matchups_remoteplatform(self): self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext)) def test_3x3_matchups_cabledinstrument(self): self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext)) def test_3x3_matchups_cablednode(self): self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext)) def base_3x3_matchups(self, deployment_context): """ This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1) """ deployment_context_type = type(deployment_context).__name__ instrument_model_id = [self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)] platform_model_id = [self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)] instrument_device_id = [self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)] platform_device_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)] instrument_site_id = [self.RR2.create(any_old(RT.InstrumentSite, {"planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="instport_%d" % (i+1))})) for i in range(9)] platform_site_id = [self.RR2.create(any_old(RT.PlatformSite, {"planned_uplink_port": IonObject(OT.PlatformPort, reference_designator="platport_%d" % (i+1))})) for i in range(4)] def instrument_model_at(platform_idx, instrument_idx): m = platform_idx * 2 if instrument_idx > 0: m += 1 return m def platform_model_at(platform_idx): if platform_idx > 0: return 1 return 0 def instrument_at(platform_idx, instrument_idx): return platform_idx * 3 + instrument_idx # set up the structure for p in range(3): m = platform_model_at(p) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[m], platform_site_id[p]) self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[m], platform_device_id[p]) self.RR2.assign_platform_device_to_platform_device_with_has_device(platform_device_id[p], platform_device_id[3]) self.RR2.assign_platform_site_to_platform_site_with_has_site(platform_site_id[p], platform_site_id[3]) for i in range(3): m = instrument_model_at(p, i) idx = instrument_at(p, i) self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id[m], instrument_site_id[idx]) self.RR2.assign_instrument_model_to_instrument_device_with_has_model(instrument_model_id[m], instrument_device_id[idx]) self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id[idx], platform_device_id[p]) self.RR2.assign_instrument_site_to_platform_site_with_has_site(instrument_site_id[idx], platform_site_id[p]) # top level models self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[2], platform_device_id[3]) self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[2], platform_site_id[3]) # verify structure for p in range(3): parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(platform_device_id[p]) self.assertEqual(platform_device_id[3], parent_id) parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(platform_site_id[p]) self.assertEqual(platform_site_id[3], parent_id) for i in range(len(platform_site_id)): self.assertEqual(self.RR2.find_platform_model_of_platform_device_using_has_model(platform_device_id[i]), self.RR2.find_platform_model_of_platform_site_using_has_model(platform_site_id[i])) for i in range(len(instrument_site_id)): self.assertEqual(self.RR2.find_instrument_model_of_instrument_device_using_has_model(instrument_device_id[i]), self.RR2.find_instrument_model_of_instrument_site_using_has_model(instrument_site_id[i])) port_assignments = {} for p in range(3): port_assignments[platform_device_id[p]] = "platport_%d" % (p+1) for i in range(3): idx = instrument_at(p, i) port_assignments[instrument_device_id[idx]] = "instport_%d" % (idx+1) deployment_id = self.RR2.create(any_old(RT.Deployment, {"context": deployment_context, "port_assignments": port_assignments})) log.debug("assigning device/site to %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[3]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[3]) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_instrument_device_with_has_deployment(deployment_id, instrument_device_id[1]) self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id[1]) elif OT.CabledNodeDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[1]) self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[1]) log.debug("activation of %s deployment", deployment_context_type) self.omsclient.activate_deployment(deployment_id) log.debug("validation of %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(d, self.RR2.find_platform_device_id_of_platform_site_using_has_device(platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual(d, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[i])) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.assertEqual(instrument_device_id[1], self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[1])) elif OT.CabledNodeDeploymentContext == deployment_context_type: expected_platforms = [1] expected_instruments = [3, 4, 5] # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual(i in expected_platforms, d in self.RR2.find_platform_device_ids_of_platform_site_using_has_device(platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual(i in expected_instruments, d in self.RR2.find_instrument_device_ids_of_instrument_site_using_has_device(instrument_site_id[i]))
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() # create missing data process definition dpd_obj = IonObject(RT.DataProcessDefinition, name=LOGICAL_TRANSFORM_DEFINITION_NAME, description="normally in preload", module='ion.processes.data.transforms.logical_transform', class_name='logical_transform') self.dataprocessclient.create_data_process_definition(dpd_obj) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip('not ready') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [parsed_config] ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) self.assertEqual(1, len(extended_product.data_processes) ) self.assertEqual(3, len(extended_product.process_input_data_products) ) # log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) # log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id) #------------------------------- # Cleanup #------------------------------- self.dpmsclient.delete_data_product(ctd_parsed_data_product) self.dpmsclient.delete_data_product(log_data_product_id) self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() config.bootstrap.use_es = True self._start_container() self.addCleanup(TestActivateInstrumentIntegration.es_cleanup) self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubcli = PubsubManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dpclient = DataProductManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() @staticmethod def es_cleanup(): es_host = CFG.get_safe('server.elasticsearch.host', 'localhost') es_port = CFG.get_safe('server.elasticsearch.port', '9200') es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10) indexes = STD_INDEXES.keys() indexes.append('%s_resources_index' % get_sys_name().lower()) indexes.append('%s_events_index' % get_sys_name().lower()) for index in indexes: IndexManagementService._es_call(es.river_couchdb_delete, index) IndexManagementService._es_call(es.index_delete, index) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name + '_logger') producer_definition.executable = { 'module': 'ion.processes.data.stream_granule_logger', 'class': 'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition( process_definition=producer_definition) configuration = { 'process': { 'stream_id': stream_id, } } pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name='', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name='notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification( notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification( notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id)) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument( self, expected_instrument_device_id='', extended_instrument=None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance( extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance( extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance( extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance( extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) self.assertEqual( extended_instrument.computed.communications_status_roll_up.value, StatusType.STATUS_WARNING) self.assertEqual( extended_instrument.computed.data_status_roll_up.value, StatusType.STATUS_OK) self.assertEqual( extended_instrument.computed.power_status_roll_up.value, StatusType.STATUS_WARNING) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value)) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_instrument_device_id) self.assertEqual(notification.origin_type, "instrument") self.assertEqual(notification.event_type, 'ResourceLifecycleEvent') def _check_computed_attributes_of_extended_product( self, expected_data_product_id='', extended_data_product=None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance( extended_data_product.computed.product_download_size_estimated, ComputedIntValue) self.assertIsInstance( extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance( extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance( extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance( extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance( extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertTrue('ok' in extended_data_product.computed.last_granule. value['quality_flag']) self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value)) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_data_product_id) self.assertEqual(notification.origin_type, "data product") self.assertEqual(notification.event_type, 'DetectionEvent') @attr('LOCOINT') @unittest.skipIf(not use_es, 'No ElasticSearch') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ # kwargs = { # 'name' : 'test_sim_warning', # 'stream_name' : 'parsed', # 'value_id' : 'temp', # 'message' : 'Temperature is above test range of 5.0.', # 'type' : StreamAlarmType.WARNING, # 'upper_bound' : 5.0, # 'upper_rel_op' : '<' # } kwargs = { 'name': 'temperature_warning_interval', 'stream_name': 'parsed', 'value_id': 'temp', 'message': 'Temperature is below the normal range of 50.0 and above.', 'type': StreamAlarmType.WARNING, 'lower_bound': 50.0, 'lower_rel_op': '<' } # Create alarm object. alarm = {} alarm['type'] = 'IntervalAlarmDef' alarm['kwargs'] = kwargs raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5) parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5, alarms=[alarm]) # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri= "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg", stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification(user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification(user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources( restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #setup a subscriber to alarm events from the device self._events_received = [] self._event_count = 0 self._samples_out_of_range = 0 self._samples_complete = False self._async_sample_result = AsyncResult() def consume_event(*args, **kwargs): log.debug( 'TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) self._event_count = len(self._events_received) self._async_sample_result.set() self._event_subscriber = EventSubscriber( event_type= 'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent callback=consume_event, origin=instDevice_id) self._event_subscriber.start() #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) def stop_subscriber(): self._event_subscriber.stop() self._event_subscriber = None self.addCleanup(stop_subscriber) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % inst_agent_instance_obj.agent_process_id) log.debug('Instrument agent instance obj: = %s', str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient( instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s", str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) self._samples_complete = True #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt.pretty_print())) temp_vals = rdt['temp'] self.assertEquals(len(temp_vals), 10) log.debug("test_activateInstrumentSample: all temp_vals: %s", temp_vals) #out_of_range_temp_vals = [i for i in temp_vals if i > 5] out_of_range_temp_vals = [i for i in temp_vals if i < 50.0] log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s", out_of_range_temp_vals) self._samples_out_of_range = len(out_of_range_temp_vals) # if no bad values were produced, then do not wait for an event if self._samples_out_of_range == 0: self._async_sample_result.set() log.debug("test_activateInstrumentSample: _events_received: %s", self._events_received) log.debug("test_activateInstrumentSample: _event_count: %s", self._event_count) self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout) replay_data = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("RDT raw: %s", str(rdt.pretty_print())) raw_vals = rdt['raw'] self.assertEquals(len(raw_vals), 10) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests. value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id1, extended_data_product=extended_product) #-------------------------------------------------------------------------------- #put some events into the eventsdb to test - this should set the comms and data status to WARNING #-------------------------------------------------------------------------------- t = get_ion_ts() self.event_publisher.publish_event(ts_created=t, event_type='DeviceStatusEvent', origin=instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values=[200]) self.event_publisher.publish_event( ts_created=t, event_type='DeviceCommsEvent', origin=instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds=20) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_1) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id2, extended_data_product=extended_product) #---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING --------- t = get_ion_ts() self.event_publisher.publish_event(ts_created=t, event_type='DeviceStatusEvent', origin=instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values=[200]) self.event_publisher.publish_event( ts_created=t, event_type='DeviceCommsEvent', origin=instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds=20) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() config.bootstrap.use_es = True self._start_container() self.addCleanup(TestActivateInstrumentIntegration.es_cleanup) self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() @staticmethod def es_cleanup(): es_host = CFG.get_safe('server.elasticsearch.host', 'localhost') es_port = CFG.get_safe('server.elasticsearch.port', '9200') es = ep.ElasticSearch( host=es_host, port=es_port, timeout=10 ) indexes = STD_INDEXES.keys() indexes.append('%s_resources_index' % get_sys_name().lower()) indexes.append('%s_events_index' % get_sys_name().lower()) for index in indexes: IndexManagementService._es_call(es.river_couchdb_delete,index) IndexManagementService._es_call(es.index_delete,index) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name = '', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name= 'notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) ) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) self.assertEqual(extended_instrument.computed.communications_status_roll_up.value, StatusType.STATUS_WARNING) self.assertEqual(extended_instrument.computed.data_status_roll_up.value, StatusType.STATUS_OK) self.assertEqual(extended_instrument.computed.power_status_roll_up.value, StatusType.STATUS_WARNING) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_instrument_device_id) self.assertEqual(notification.origin_type, "instrument") self.assertEqual(notification.event_type, 'ResourceLifecycleEvent') def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertTrue( 'ok' in extended_data_product.computed.last_granule.value['quality_flag'] ) self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) ) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(notification.origin, expected_data_product_id) self.assertEqual(notification.origin_type, "data product") self.assertEqual(notification.event_type, 'DetectionEvent') @attr('LOCOINT') @unittest.skipIf(not use_es, 'No ElasticSearch') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ # kwargs = { # 'name' : 'test_sim_warning', # 'stream_name' : 'parsed', # 'value_id' : 'temp', # 'message' : 'Temperature is above test range of 5.0.', # 'type' : StreamAlarmType.WARNING, # 'upper_bound' : 5.0, # 'upper_rel_op' : '<' # } kwargs = { 'name' : 'temperature_warning_interval', 'stream_name' : 'parsed', 'value_id' : 'temp', 'message' : 'Temperature is below the normal range of 50.0 and above.', 'type' : StreamAlarmType.WARNING, 'lower_bound' : 50.0, 'lower_rel_op' : '<' } # Create alarm object. alarm = {} alarm['type'] = 'IntervalAlarmDef' alarm['kwargs'] = kwargs raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5, alarms=[alarm] ) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg", stream_configurations = [raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s' , data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s' , dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s' , str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s' , dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources(restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #setup a subscriber to alarm events from the device self._events_received= [] self._event_count = 0 self._samples_out_of_range = 0 self._samples_complete = False self._async_sample_result = AsyncResult() def consume_event(*args, **kwargs): log.debug('TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) self._event_count = len(self._events_received) self._async_sample_result.set() self._event_subscriber = EventSubscriber( event_type= 'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent callback=consume_event, origin=instDevice_id) self._event_subscriber.start() #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) def stop_subscriber(): self._event_subscriber.stop() self._event_subscriber = None self.addCleanup(stop_subscriber) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % inst_agent_instance_obj.agent_process_id) log.debug('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s" , str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s" , str(retval)) log.debug( "test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s" , str(reply)) self._samples_complete = True #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt.pretty_print()) ) temp_vals = rdt['temp'] self.assertEquals(len(temp_vals) , 10) log.debug("test_activateInstrumentSample: all temp_vals: %s", temp_vals ) #out_of_range_temp_vals = [i for i in temp_vals if i > 5] out_of_range_temp_vals = [i for i in temp_vals if i < 50.0] log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s", out_of_range_temp_vals ) self._samples_out_of_range = len(out_of_range_temp_vals) # if no bad values were produced, then do not wait for an event if self._samples_out_of_range == 0: self._async_sample_result.set() log.debug("test_activateInstrumentSample: _events_received: %s", self._events_received ) log.debug("test_activateInstrumentSample: _event_count: %s", self._event_count ) self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout) replay_data = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("RDT raw: %s", str(rdt.pretty_print()) ) raw_vals = rdt['raw'] self.assertEquals(len(raw_vals) , 10) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests.value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product) #-------------------------------------------------------------------------------- #put some events into the eventsdb to test - this should set the comms and data status to WARNING #-------------------------------------------------------------------------------- t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent', origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] ) self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent', origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 ) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1) self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product) #---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING --------- t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceStatusEvent', origin = instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values = [200] ) self.event_publisher.publish_event( ts_created= t, event_type = 'DeviceCommsEvent', origin = instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds = 20 ) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url("res/deploy/r2deploy.yml") # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) # setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] def create_logger(self, name, stream_id=""): # logger process producer_definition = ProcessDefinition(name=name + "_logger") producer_definition.executable = { "module": "ion.processes.data.stream_granule_logger", "class": "StreamGranuleLogger", } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = {"process": {"stream_id": stream_id}} pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration ) return pid @unittest.skip("TBD") def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject( RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) log.debug("new InstrumentModel id = %s ", instModel_id) # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name="agent007", description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent", ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.debug("new InstrumentAgent id = %s", instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug( "test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) " ) instDevice_obj = IonObject( RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id ) instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstance", description="SBE37IMAgentInstance", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", comms_device_address="sbe37-simulator.oceanobservatories.org", comms_device_port=4001, port_agent_work_dir="/tmp/", port_agent_delimeter=["<<", ">>"], ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id ) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def) log.debug("new Stream Definition id = %s", instDevice_id) log.debug("Creating new CDM data product with a stream definition") craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject( RT.DataProduct, name="the parsed data", description="ctd stream test", temporal_domain=tdom, spatial_domain=sdom, ) data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug("new dp_id = %s", data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug("Data product streams1 = %s", stream_ids) pid = self.create_logger("ctd_parsed", stream_ids[0]) self.loggerpids.append(pid) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition # ------------------------------- log.debug("test_activateInstrumentSample: create data process definition ctd_L0_all") dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L0_all", description="transform ctd package into three separate L0 streams", module="ion.processes.data.transforms.ctd.ctd_L0_all", class_name="ctd_L0_all", process_source="some_source_reference", ) try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" % ex) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products # ------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubcli.create_stream_definition( container=outgoing_stream_l0_conductivity, name="L0_Conductivity" ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubcli.create_stream_definition( container=outgoing_stream_l0_pressure, name="L0_Pressure" ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubcli.create_stream_definition( container=outgoing_stream_l0_temperature, name="L0_Temperature" ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products = {} log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name="L0_Conductivity", description="transform output conductivity", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id, parameter_dictionary ) self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) stream_ids, _ = self.rrclient.find_objects(ctd_l0_conductivity_output_dp_id, PRED.hasStream, None, True) log.debug(" ctd_l0_conductivity stream id = %s", str(stream_ids)) pid = self.create_logger(" ctd_l1_conductivity", stream_ids[0]) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name="L0_Pressure", description="transform output pressure", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary ) self.output_products["pressure"] = ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) stream_ids, _ = self.rrclient.find_objects(ctd_l0_pressure_output_dp_id, PRED.hasStream, None, True) log.debug(" ctd_l0_pressure stream id = %s", str(stream_ids)) pid = self.create_logger(" ctd_l0_pressure", stream_ids[0]) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name="L0_Temperature", description="transform output temperature", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary ) self.output_products["temperature"] = ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) stream_ids, _ = self.rrclient.find_objects(ctd_l0_temperature_output_dp_id, PRED.hasStream, None, True) log.debug(" ctd_l0_temperature stream id = %s", str(stream_ids)) pid = self.create_logger(" ctd_l0_temperature", stream_ids[0]) self.loggerpids.append(pid) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process # ------------------------------- log.debug("test_activateInstrumentSample: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, [data_product_id1], self.output_products ) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return") log.debug("Creating new RAW data product with a stream definition") raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def) dp_obj = IonObject( RT.DataProduct, name="the raw data", description="raw stream test", temporal_domain=tdom, spatial_domain=sdom, ) data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary) log.debug("new dp_id = %s", str(data_product_id2)) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug("Data product streams2 = %s", str(stream_ids)) self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) log.debug("Instrument agent instance obj: = %s", str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command="initialize") retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s", str(retval)) time.sleep(1) log.debug("test_activateInstrumentSample: Sending go_active command (L4-CI-SA-RQ-334)") cmd = AgentCommand(command="go_active") reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) time.sleep(1) cmd = AgentCommand(command="get_current_state") retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "test_activateInstrumentSample: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state), ) cmd = AgentCommand(command="run") reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) time.sleep(1) log.debug("test_activateInstrumentSample: calling acquire_sample ") cmd = AgentCommand(command="acquire_sample") reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentSample: return from acquire_sample %s", str(reply)) time.sleep(1) log.debug("test_activateInstrumentSample: calling acquire_sample 2") cmd = AgentCommand(command="acquire_sample") reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentSample: return from acquire_sample 2 %s", str(reply)) time.sleep(1) log.debug("test_activateInstrumentSample: calling acquire_sample 3") cmd = AgentCommand(command="acquire_sample") reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentSample: return from acquire_sample 3 %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command="reset") reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(1) # ------------------------------- # Deactivate InstrumentAgentInstance # ------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) @unittest.skip("TBD") def test_activateInstrumentStream(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject( RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) log.debug("new InstrumentModel id = %s ", instModel_id) # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name="agent007", description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent", ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.debug("new InstrumentAgent id = %s", instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug( "test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) " ) instDevice_obj = IonObject( RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id ) instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstance", description="SBE37IMAgentInstance", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", comms_device_address="sbe37-simulator.oceanobservatories.org", comms_device_port=4001, port_agent_work_dir="/tmp/", port_agent_delimeter=["<<", ">>"], ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id ) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def) log.debug("new Stream Definition id = %s", instDevice_id) log.debug("Creating new CDM data product with a stream definition") craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject( RT.DataProduct, name="the parsed data", description="ctd stream test", temporal_domain=tdom, spatial_domain=sdom, ) data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug("new dp_id = %s", data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug("Data product streams1 = %s", stream_ids) pid = self.create_logger("ctd_parsed", stream_ids[0]) self.loggerpids.append(pid) # simdata_subscription_id = self.pubsubcli.create_subscription( # query=StreamQuery([stream_ids[0]]), # exchange_name='Sim_data_queue', # name='SimDataSubscription', # description='SimData SubscriptionDescription' # ) # # # def simdata_message_received(message, headers): # input = str(message) # log.debug("test_activateInstrumentStream: granule received: %s", input) # # # subscriber_registrar = StreamSubscriberRegistrar(process=self.container, container=self.container) # simdata_subscriber = subscriber_registrar.create_subscriber(exchange_name='Sim_data_queue', callback=simdata_message_received) # # # Start subscribers # simdata_subscriber.start() # # # Activate subscriptions # self.pubsubcli.activate_subscription(simdata_subscription_id) log.debug("Creating new RAW data product with a stream definition") raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def) dp_obj = IonObject( RT.DataProduct, name="the raw data", description="raw stream test", temporal_domain=tdom, spatial_domain=sdom, ) data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary) log.debug("new dp_id = %s", str(data_product_id2)) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug("Data product streams2 = %s", str(stream_ids)) self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) log.debug("test_activateInstrumentStream Instrument agent instance obj: = %s", str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess()) log.debug("test_activateInstrumentStream: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command="initialize") retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: initialize %s", str(retval)) time.sleep(2) log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)") cmd = AgentCommand(command="go_active") reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command="get_current_state") retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state), ) cmd = AgentCommand(command="run") reply = self._ia_client.execute_agent(cmd) time.sleep(2) cmd = AgentCommand(command="get_current_state") retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from run state: %s", str(state)) # Make sure the sampling rate and transmission are sane. params = {SBE37Parameter.NAVG: 1, SBE37Parameter.INTERVAL: 5, SBE37Parameter.TXREALTIME: True} self._ia_client.set_param(params) time.sleep(2) log.debug("test_activateInstrumentStream: calling go_streaming ") cmd = AgentCommand(command="go_streaming") reply = self._ia_client.execute_agent(cmd) time.sleep(2) cmd = AgentCommand(command="get_current_state") retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state)) time.sleep(5) log.debug("test_activateInstrumentStream: calling go_observatory") cmd = AgentCommand(command="go_observatory") reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) log.debug("test_activateInstrumentStream: calling reset ") cmd = AgentCommand(command="reset") reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result)) time.sleep(2) # ------------------------------- # Deactivate InstrumentAgentInstance # ------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid)
class TestDeployment(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dmpsclient = DataProductManagementServiceClient( node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.psmsclient = PubsubManagementServiceClient( node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.c = DotDict() self.c.resource_registry = self.rrclient self.RR2 = EnhancedResourceRegistryClient(self.rrclient) self.dsmsclient = DataProcessManagementServiceClient( node=self.container.node) # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources( RT.DataProcess, None, None, True)[0]: self.dsmsclient.deactivate_data_process(proc_id) self.dsmsclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) #@unittest.skip("targeting") def test_create_deployment(self): #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) #retrieve the deployment objects and check that the assoc site and device are attached read_deployment_obj = self.omsclient.read_deployment(deployment_id) log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj)) site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(site_ids), 1) device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True) self.assertEqual(len(device_ids), 1) #delete the deployment self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def test_prepare_deployment_support(self): deploy_sup = self.omsclient.prepare_deployment_support() self.assertTrue(deploy_sup) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].type_, "AssocDeploymentInstDevice") self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice'].type_, "AssocDeploymentPlatDevice") self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].type_, "AssocDeploymentInstSite") self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite'].type_, "AssocDeploymentPlatSite") self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite']. associated_resources, []) #create a deployment with metadata and an initial site and device platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site') site_id = self.omsclient.create_platform_site(platform_site__obj) platform_device__obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') device_id = self.imsclient.create_platform_device(platform_device__obj) start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2014, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformDevice']. resources), 1) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformDevice']. associated_resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformSite'].resources ), 1) self.assertEquals( deploy_sup.associations['DeploymentHasPlatformSite']. associated_resources, []) self.omsclient.assign_site_to_deployment(site_id, deployment_id) self.omsclient.assign_device_to_deployment(device_id, deployment_id) deploy_sup = self.omsclient.prepare_deployment_support(deployment_id) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentDevice']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformDevice']. resources), 1) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformDevice']. associated_resources), 1) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite'].resources, []) self.assertEquals( deploy_sup.associations['DeploymentHasInstrumentSite']. associated_resources, []) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformSite'].resources ), 1) self.assertEquals( len(deploy_sup.associations['DeploymentHasPlatformSite']. associated_resources), 1) #delete the deployment self.omsclient.force_delete_deployment(deployment_id) # now try to get the deleted dp object try: self.omsclient.read_deployment(deployment_id) except NotFound: pass else: self.fail("deleted deployment was found during read") #@unittest.skip("targeting") def base_activate_deployment(self, make_assigns=False): # Create platform site, platform device, platform model bounds = GeospatialBounds(geospatial_latitude_limit_north=float(5), geospatial_latitude_limit_south=float(5), geospatial_longitude_limit_west=float(15), geospatial_longitude_limit_east=float(15), geospatial_vertical_min=float(0), geospatial_vertical_max=float(1000)) platform_site__obj = IonObject(RT.PlatformSite, name='PlatformSite1', description='test platform site', constraint_list=[bounds]) platform_site_id = self.omsclient.create_platform_site( platform_site__obj) platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice1', description='test platform device') platform_device_id = self.imsclient.create_platform_device( platform_device_obj) platform_model__obj = IonObject(RT.PlatformModel, name='PlatformModel1', description='test platform model') platform_model_id = self.imsclient.create_platform_model( platform_model__obj) # Create instrument site #------------------------------------------------------------------------------------- bounds = GeospatialBounds(geospatial_latitude_limit_north=float(45), geospatial_latitude_limit_south=float(40), geospatial_longitude_limit_west=float(-75), geospatial_longitude_limit_east=float(-70), geospatial_vertical_min=float(0), geospatial_vertical_max=float(500)) instrument_site_obj = IonObject( RT.InstrumentSite, name='InstrumentSite1', description='test instrument site', reference_designator='GA01SUMO-FI003-01-CTDMO0999', constraint_list=[bounds]) instrument_site_id = self.omsclient.create_instrument_site( instrument_site_obj, platform_site_id) pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.psmsclient.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) # Create an instrument device instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice1', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) pp_obj = IonObject(OT.PlatformPort, reference_designator='GA01SUMO-FI003-01-CTDMO0999', port_type=PortTypeEnum.PAYLOAD, ip_address='1') port_assignments = {instrument_device_id: pp_obj} #---------------------------------------------------------------------------------------------------- # Create an instrument model instrument_model_obj = IonObject(RT.InstrumentModel, name='InstrumentModel1', description='test instrument model') instrument_model_id = self.imsclient.create_instrument_model( instrument_model_obj) # Create a deployment object #---------------------------------------------------------------------------------------------------- start = str(int(time.mktime(datetime.datetime(2013, 1, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2020, 1, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment', context=IonObject( OT.CabledNodeDeploymentContext), port_assignments=port_assignments, constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) if make_assigns: self.imsclient.assign_platform_model_to_platform_device( platform_model_id, platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device( instrument_model_id, instrument_device_id) self.omsclient.assign_platform_model_to_platform_site( platform_model_id, platform_site_id) self.omsclient.assign_instrument_model_to_instrument_site( instrument_model_id, instrument_site_id) self.omsclient.assign_site_to_deployment(platform_site_id, deployment_id) self.omsclient.assign_device_to_deployment(platform_device_id, deployment_id) ret = DotDict(instrument_site_id=instrument_site_id, instrument_device_id=instrument_device_id, instrument_model_id=instrument_model_id, platform_site_id=platform_site_id, platform_device_id=platform_device_id, platform_model_id=platform_model_id, deployment_id=deployment_id) return ret def _create_subsequent_deployment(self, prior_dep_info): platform_device_obj = IonObject(RT.PlatformDevice, name='PlatformDevice2', description='test platform device') platform_device_id = self.imsclient.create_platform_device( platform_device_obj) instrument_device_obj = IonObject(RT.InstrumentDevice, name='InstrumentDevice2', description='test instrument device') instrument_device_id = self.imsclient.create_instrument_device( instrument_device_obj) self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) self.imsclient.assign_platform_model_to_platform_device( prior_dep_info.platform_model_id, platform_device_id) self.imsclient.assign_instrument_model_to_instrument_device( prior_dep_info.instrument_model_id, instrument_device_id) start = str(int(time.mktime(datetime.datetime(2013, 6, 1).timetuple()))) end = str(int(time.mktime(datetime.datetime(2020, 6, 1).timetuple()))) temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start, end_datetime=end) deployment_obj = IonObject(RT.Deployment, name='TestDeployment2', description='some new deployment', context=IonObject( OT.CabledNodeDeploymentContext), constraint_list=[temporal_bounds]) deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.assign_site_to_deployment( prior_dep_info.platform_site_id, deployment_id) self.omsclient.assign_device_to_deployment( prior_dep_info.platform_device_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id)) ret = DotDict(instrument_device_id=instrument_device_id, platform_device_id=platform_device_id, deployment_id=deployment_id) return ret #@unittest.skip("targeting") def test_activate_deployment_normal(self): res = self.base_activate_deployment(make_assigns=True) before_activate_instrument_device_obj = self.rrclient.read( res.instrument_device_id) self.assertNotEquals(before_activate_instrument_device_obj.lcstate, LCS.DEPLOYED) log.debug("activating deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) def assertGeospatialBoundsEquals(a, b): self.assertEquals(a['geospatial_latitude_limit_north'], b['geospatial_latitude_limit_north']) self.assertEquals(a['geospatial_latitude_limit_south'], b['geospatial_latitude_limit_south']) self.assertEquals(a['geospatial_longitude_limit_west'], b['geospatial_longitude_limit_west']) self.assertEquals(a['geospatial_longitude_limit_east'], b['geospatial_longitude_limit_east']) def assertGeospatialBoundsNotEquals(a, b): self.assertNotEquals(a['geospatial_latitude_limit_north'], b['geospatial_latitude_limit_north']) self.assertNotEquals(a['geospatial_latitude_limit_south'], b['geospatial_latitude_limit_south']) self.assertNotEquals(a['geospatial_longitude_limit_west'], b['geospatial_longitude_limit_west']) self.assertNotEquals(a['geospatial_longitude_limit_east'], b['geospatial_longitude_limit_east']) after_activate_instrument_device_obj = self.rrclient.read( res.instrument_device_id) assertGeospatialBoundsNotEquals( before_activate_instrument_device_obj.geospatial_bounds, after_activate_instrument_device_obj.geospatial_bounds) deployment_obj = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj.lcstate, LCS.DEPLOYED) log.debug("deactivatin deployment, expecting success") self.omsclient.deactivate_deployment(res.deployment_id) after_deactivate_instrument_device_obj = self.rrclient.read( res.instrument_device_id) assertGeospatialBoundsNotEquals( after_activate_instrument_device_obj.geospatial_bounds, after_deactivate_instrument_device_obj.geospatial_bounds) deployment_obj = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj.lcstate, LCS.INTEGRATED) def test_activate_deployment_redeploy(self): dep_util = DeploymentUtil(self.container) res = self.base_activate_deployment(make_assigns=True) log.debug("activating first deployment, expecting success") self.omsclient.activate_deployment(res.deployment_id) deployment_obj1 = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj1.lcstate, LCS.DEPLOYED) next_dep_info = self._create_subsequent_deployment(res) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertNotEquals(deployment_obj2.lcstate, LCS.DEPLOYED) log.debug("activating subsequent deployment, expecting success") self.omsclient.activate_deployment(next_dep_info.deployment_id) deployment_obj1 = self.RR2.read(res.deployment_id) self.assertEquals(deployment_obj1.lcstate, LCS.INTEGRATED) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertEquals(deployment_obj2.lcstate, LCS.DEPLOYED) dep1_tc = dep_util.get_temporal_constraint(deployment_obj1) dep2_tc = dep_util.get_temporal_constraint(deployment_obj2) self.assertLessEqual(float(dep1_tc.end_datetime), float(dep2_tc.end_datetime)) log.debug("deactivating second deployment, expecting success") self.omsclient.deactivate_deployment(next_dep_info.deployment_id) deployment_obj2 = self.RR2.read(next_dep_info.deployment_id) self.assertEquals(deployment_obj2.lcstate, LCS.INTEGRATED) #@unittest.skip("targeting") def test_activate_deployment_nomodels(self): res = self.base_activate_deployment() self.omsclient.assign_site_to_deployment(res.platform_site_id, res.deployment_id) self.omsclient.assign_device_to_deployment(res.platform_device_id, res.deployment_id) log.debug( "activating deployment without site+device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") log.debug("assigning instrument site model") self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug( "activating deployment without device models, expecting fail") self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1") #@unittest.skip("targeting") def test_activate_deployment_nosite(self): res = self.base_activate_deployment() log.debug("assigning instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument device only") self.omsclient.assign_device_to_deployment(res.instrument_device_id, res.deployment_id) log.debug( "activating deployment without instrument site, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest) #@unittest.skip("targeting") def test_activate_deployment_nodevice(self): res = self.base_activate_deployment() log.debug("assigning platform and instrument models") self.imsclient.assign_instrument_model_to_instrument_device( res.instrument_model_id, res.instrument_device_id) self.omsclient.assign_instrument_model_to_instrument_site( res.instrument_model_id, res.instrument_site_id) log.debug("deploying instrument site only") self.omsclient.assign_site_to_deployment(res.instrument_site_id, res.deployment_id) log.debug("activating deployment without device, expecting fail") self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment") def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message=""): with self.assertRaises(err_type) as cm: self.omsclient.activate_deployment(deployment_id) log.debug("assert_deploy_fail cm: %s", str(cm)) if fail_message: self.assertIn(fail_message, cm.exception.message) def test_3x3_matchups_remoteplatform(self): self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext)) def test_3x3_matchups_cabledinstrument(self): self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext)) def test_3x3_matchups_cablednode(self): self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext)) def base_3x3_matchups(self, deployment_context): """ This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1) """ deployment_context_type = type(deployment_context).__name__ instrument_model_id = [ self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6) ] platform_model_id = [ self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3) ] instrument_device_id = [ self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9) ] platform_device_id = [ self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4) ] instrument_site_id = [ self.RR2.create( any_old( RT.InstrumentSite, { "reference_designator": "GA01SUMO-FI003-0%s-CTDMO0999" % (i + 1), "planned_uplink_port": IonObject( OT.PlatformPort, reference_designator="GA01SUMO-FI003-0%s-CTDMO0999" % (i + 1)) })) for i in range(9) ] platform_site_id = [ self.RR2.create( any_old( RT.PlatformSite, { "reference_designator": "GA01SUMO-FI003-0%s-CTDMO0888" % (i + 1), "planned_uplink_port": IonObject( OT.PlatformPort, reference_designator="GA01SUMO-FI003-0%s-CTDMO0888" % (i + 1)) })) for i in range(4) ] def instrument_model_at(platform_idx, instrument_idx): m = platform_idx * 2 if instrument_idx > 0: m += 1 return m def platform_model_at(platform_idx): if platform_idx > 0: return 1 return 0 def instrument_at(platform_idx, instrument_idx): return platform_idx * 3 + instrument_idx # set up the structure for p in range(3): m = platform_model_at(p) self.RR2.assign_platform_model_to_platform_site_with_has_model( platform_model_id[m], platform_site_id[p]) self.RR2.assign_platform_model_to_platform_device_with_has_model( platform_model_id[m], platform_device_id[p]) self.RR2.assign_platform_device_to_platform_device_with_has_device( platform_device_id[p], platform_device_id[3]) self.RR2.assign_platform_site_to_platform_site_with_has_site( platform_site_id[p], platform_site_id[3]) for i in range(3): m = instrument_model_at(p, i) idx = instrument_at(p, i) self.RR2.assign_instrument_model_to_instrument_site_with_has_model( instrument_model_id[m], instrument_site_id[idx]) self.RR2.assign_instrument_model_to_instrument_device_with_has_model( instrument_model_id[m], instrument_device_id[idx]) self.RR2.assign_instrument_device_to_platform_device_with_has_device( instrument_device_id[idx], platform_device_id[p]) self.RR2.assign_instrument_site_to_platform_site_with_has_site( instrument_site_id[idx], platform_site_id[p]) # top level models self.RR2.assign_platform_model_to_platform_device_with_has_model( platform_model_id[2], platform_device_id[3]) self.RR2.assign_platform_model_to_platform_site_with_has_model( platform_model_id[2], platform_site_id[3]) # verify structure for p in range(3): parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device( platform_device_id[p]) self.assertEqual(platform_device_id[3], parent_id) parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site( platform_site_id[p]) self.assertEqual(platform_site_id[3], parent_id) for i in range(len(platform_site_id)): self.assertEqual( self.RR2. find_platform_model_of_platform_device_using_has_model( platform_device_id[i]), self.RR2.find_platform_model_of_platform_site_using_has_model( platform_site_id[i])) for i in range(len(instrument_site_id)): self.assertEqual( self.RR2. find_instrument_model_of_instrument_device_using_has_model( instrument_device_id[i]), self.RR2. find_instrument_model_of_instrument_site_using_has_model( instrument_site_id[i])) # OOIReferenceDesignator format: GA01SUMO-FI003-03-CTDMO0999 (site-platform_id-port-device_id) port_assignments = {} for p in range(3): ref_desig = "GA01SUMO-FI003-0%s-CTDMO0888" % (p + 1) pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type=PortTypeEnum.PAYLOAD, ip_address=str(p)) port_assignments[platform_device_id[p]] = pp_obj for i in range(3): ref_desig = "GA01SUMO-FI003-0%s-CTDMO0999" % ((p * 3) + i + 1) pp_obj = IonObject(OT.PlatformPort, reference_designator=ref_desig, port_type=PortTypeEnum.PAYLOAD, ip_address=str(p)) idx = instrument_at(p, i) port_assignments[instrument_device_id[idx]] = pp_obj deployment_id = self.RR2.create( any_old( RT.Deployment, { "context": deployment_context, "port_assignments": port_assignments })) log.debug("assigning device/site to %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment( deployment_id, platform_device_id[3]) self.RR2.assign_deployment_to_platform_site_with_has_deployment( deployment_id, platform_site_id[3]) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_instrument_device_with_has_deployment( deployment_id, instrument_device_id[1]) self.RR2.assign_deployment_to_instrument_site_with_has_deployment( deployment_id, instrument_site_id[1]) elif OT.CabledNodeDeploymentContext == deployment_context_type: self.RR2.assign_deployment_to_platform_device_with_has_deployment( deployment_id, platform_device_id[1]) self.RR2.assign_deployment_to_platform_site_with_has_deployment( deployment_id, platform_site_id[1]) log.debug("activation of %s deployment", deployment_context_type) self.omsclient.activate_deployment(deployment_id) log.debug("validation of %s deployment", deployment_context_type) if OT.RemotePlatformDeploymentContext == deployment_context_type: # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual( d, self.RR2. find_platform_device_id_of_platform_site_using_has_device( platform_site_id[i])) for i, d in enumerate(instrument_device_id): self.assertEqual( d, self.RR2. find_instrument_device_id_of_instrument_site_using_has_device( instrument_site_id[i])) elif OT.CabledInstrumentDeploymentContext == deployment_context_type: self.assertEqual( instrument_device_id[1], self.RR2. find_instrument_device_id_of_instrument_site_using_has_device( instrument_site_id[1])) elif OT.CabledNodeDeploymentContext == deployment_context_type: expected_platforms = [1] # verify proper associations for i, d in enumerate(platform_device_id): self.assertEqual( i in expected_platforms, d in self.RR2. find_platform_device_ids_of_platform_site_using_has_device( platform_site_id[i]))
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() # self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.container.start_rel_from_url("res/deploy/r2deploy.yml") print "started services" # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_logger(self, name, stream_id=""): # logger process producer_definition = ProcessDefinition(name=name + "_logger") producer_definition.executable = { "module": "ion.processes.data.stream_granule_logger", "class": "StreamGranuleLogger", } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = {"process": {"stream_id": stream_id}} pid = self.processdispatchclient.schedule_process( process_definition_id=logger_procdef_id, configuration=configuration ) return pid def cleanupprocs(self): stm = os.popen("ps -e | grep ion.agents.port.logger_process") procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) stm = os.popen("ps -e | grep ion.agents.instrument.zmq_driver_process") procs = stm.read() if len(procs) > 0: procs = procs.split() if procs[0].isdigit(): pid = int(procs[0]) os.kill(pid, signal.SIGKILL) # stm = os.popen('rm /tmp/*.pid.txt') @unittest.skip("Deprecated by IngestionManagement refactor, timeout on start inst agent?") def test_deploy_activate_full(self): # ensure no processes or pids are left around by agents or Sims # self.cleanupprocs() self.loggerpids = [] # ------------------------------- # Create InstrumentModel # ------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) # ------------------------------- # Create InstrumentAgent # ------------------------------- instAgent_obj = IonObject( RT.InstrumentAgent, name="agent007", description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) log.debug("new InstrumentAgent id = %s", instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # ------------------------------- # Create Instrument Site # ------------------------------- instrumentSite_obj = IonObject(RT.InstrumentSite, name="instrumentSite1", description="SBE37IMInstrumentSite") try: instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id="") except BadRequest as ex: self.fail("failed to create new InstrumentSite: %s" % ex) print "test_deployAsPrimaryDevice: new instrumentSite id = ", instrumentSite_id self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id) # ------------------------------- # Logical Transform: Output Data Products # ------------------------------- # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( "ctd_parsed_param_dict", id_only=True ) parsed_stream_def_id = self.pubsubclient.create_stream_definition( name="parsed", parameter_dictionary_id=parsed_pdict_id ) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_raw_param_dict", id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition(name="raw", parameter_dictionary_id=raw_pdict_id) # ------------------------------- # Create Old InstrumentDevice # ------------------------------- instDevice_obj = IonObject( RT.InstrumentDevice, name="SBE37IMDeviceYear1", description="SBE37IMDevice for the FIRST year of deployment", serial_number="12345", ) try: oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print "test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ", oldInstDevice_id self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE) # ------------------------------- # Create Raw and Parsed Data Products for the device # ------------------------------- dp_obj = IonObject( RT.DataProduct, name="SiteDataProduct", description="SiteDataProduct", temporal_domain=tdom, spatial_domain=sdom, ) instrument_site_output_dp_id = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id ) self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=instrument_site_output_dp_id ) # self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasStream, None, True) log.debug("Data product streams1 = %s", stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True) log.debug("Data set for data_product_id1 = %s", dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger("ctd_parsed", stream_ids[0]) self.loggerpids.append(pid) self.omsclient.create_site_data_product(instrumentSite_id, instrument_site_output_dp_id) # ------------------------------- # Create Old Deployment # ------------------------------- deployment_obj = IonObject(RT.Deployment, name="first deployment") oldDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id) # ------------------------------- # Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information # cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, # ------------------------------- port_agent_config = { "device_addr": CFG.device.sbe37.host, "device_port": CFG.device.sbe37.port, "process_type": PortAgentProcessType.UNIX, "binary_path": "port_agent", "port_agent_addr": "localhost", "command_port": CFG.device.sbe37.port_agent_cmd_port, "data_port": CFG.device.sbe37.port_agent_data_port, "log_level": 5, "type": PortAgentType.ETHERNET, } raw_config = StreamConfiguration( stream_name="raw", parameter_dictionary_name="ctd_raw_param_dict", records_per_granule=2, granule_publish_rate=5, ) parsed_config = StreamConfiguration( stream_name="parsed", parameter_dictionary_name="ctd_parsed_param_dict", records_per_granule=2, granule_publish_rate=5, ) instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstanceYear1", description="SBE37IMAgentInstanceYear1", comms_device_address="sbe37-simulator.oceanobservatories.org", comms_device_port=4001, port_agent_config=port_agent_config, stream_configurations=[raw_config, parsed_config], ) oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, oldInstDevice_id ) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() # ------------------------------- # Create CTD Parsed as the Year 1 data product and attach to instrument # ------------------------------- print "Creating new CDM data product with a stream definition" dp_obj = IonObject( RT.DataProduct, name="ctd_parsed_year1", description="ctd stream test year 1", temporal_domain=tdom, spatial_domain=sdom, ) ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id ) print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year1 self.damsclient.assign_data_product( input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1 ) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True) print "test_deployAsPrimaryDevice: Data product streams1 = ", stream_ids # ------------------------------- # Create New InstrumentDevice # ------------------------------- instDevice_obj_2 = IonObject( RT.InstrumentDevice, name="SBE37IMDeviceYear2", description="SBE37IMDevice for the SECOND year of deployment", serial_number="67890", ) try: newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print "test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ", newInstDevice_id # set the LCSTATE self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE) instDevice_obj_2 = self.rrclient.read(newInstDevice_id) log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate)) # ------------------------------- # Create Old Deployment # ------------------------------- deployment_obj = IonObject(RT.Deployment, name="second deployment") newDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id) # ------------------------------- # Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information # ------------------------------- port_agent_config = { "device_addr": "sbe37-simulator.oceanobservatories.org", "device_port": 4004, "process_type": PortAgentProcessType.UNIX, "binary_path": "port_agent", "port_agent_addr": "localhost", "command_port": 4005, "data_port": 4006, "log_level": 5, "type": PortAgentType.ETHERNET, } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name="SBE37IMAgentInstanceYear2", description="SBE37IMAgentInstanceYear2", comms_device_address="sbe37-simulator.oceanobservatories.org", comms_device_port=4004, port_agent_config=port_agent_config, ) newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, newInstDevice_id ) # ------------------------------- # Create CTD Parsed as the Year 2 data product # ------------------------------- dp_obj = IonObject( RT.DataProduct, name="ctd_parsed_year2", description="ctd stream test year 2", temporal_domain=tdom, spatial_domain=sdom, ) ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id ) print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year2 self.damsclient.assign_data_product( input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2 ) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True) print "test_deployAsPrimaryDevice: Data product streams2 = ", stream_ids # ------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition # ------------------------------- log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all") dpd_obj = IonObject( RT.DataProcessDefinition, name="ctd_L0_all", description="transform ctd package into three separate L0 streams", module="ion.processes.data.transforms.ctd.ctd_L0_all", class_name="ctd_L0_all", ) try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" % ex) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products # ------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( name="L0_Conductivity", parameter_dictionary_id=parsed_pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding="conductivity" ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( name="L0_Pressure", parameter_dictionary_id=parsed_pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding="pressure" ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( name="L0_Temperature", parameter_dictionary_id=parsed_pdict_id ) self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding="temperature" ) self.output_products = {} log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name="L0_Conductivity", description="transform output conductivity", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_conductivity_output_dp_obj, stream_definition_id=parsed_stream_def_id ) self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name="L0_Pressure", description="transform output pressure", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_pressure_output_dp_obj, stream_definition_id=parsed_stream_def_id ) self.output_products["pressure"] = ctd_l0_pressure_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name="L0_Temperature", description="transform output temperature", temporal_domain=tdom, spatial_domain=sdom, ) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( data_product=ctd_l0_temperature_output_dp_obj, stream_definition_id=parsed_stream_def_id ) self.output_products["temperature"] = ctd_l0_temperature_output_dp_id # self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) # ------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product) # ------------------------------- log.debug("test_deployAsPrimaryDevice: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products ) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug("test_deployAsPrimaryDevice: create L0 all data_process return") # -------------------------------- # Activate the deployment # -------------------------------- self.omsclient.activate_deployment(oldDeployment_id) # ------------------------------- # Launch InstrumentAgentInstance Sim1, connect to the resource agent client # ------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id) self.addCleanup( self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=oldInstAgentInstance_id ) # wait for start instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) gate = ProcessStateGate( self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING ) self.assertTrue( gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id, ) inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent1_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim1 = ResourceAgentClient( "iaclient Sim1", name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess() ) print "activate_instrument: got _ia_client_sim1 %s", self._ia_client_sim1 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1)) # ------------------------------- # Launch InstrumentAgentInstance Sim2, connect to the resource agent client # ------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id) self.addCleanup( self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=newInstAgentInstance_id ) # wait for start instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) gate = ProcessStateGate( self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING ) self.assertTrue( gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id, ) inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent2_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim2 = ResourceAgentClient( "iaclient Sim2", name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess() ) print "activate_instrument: got _ia_client_sim2 %s", self._ia_client_sim2 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2)) # ------------------------------- # Streaming Sim1 (old instrument) # ------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active %s", str(reply)) self.assertTrue(reply) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim1.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE: %s", str(retval)) # ------------------------------- # Streaming Sim 2 (new instrument) # ------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active_sim2 %s", str(reply)) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim2.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s", str(retval)) gevent.sleep(10) # ------------------------------- # Shutdown Sim1 (old instrument) # ------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(5) # ------------------------------- # Shutdown Sim2 (old instrument) # ------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset_sim2 ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset_sim2 %s", str(reply)) time.sleep(5)
class BaseIntTestPlatform(IonIntegrationTestCase, HelperTestMixin): """ A base class with several conveniences supporting specific platform agent integration tests, see: - ion/agents/platform/test/test_platform_agent_with_rsn.py - ion/services/sa/observatory/test/test_platform_launch.py The platform IDs used here are organized as follows: Node1D -> MJ01C -> LJ01D where -> goes from parent platform to child platform. This is a subset of the whole topology defined in the simulated platform network (network.yml), which in turn is used by the RSN OMS simulator. - 'LJ01D' is the root platform used in test_single_platform - 'Node1D' is the root platform used in test_hierarchy Methods are provided to construct specific platform topologies, but subclasses decide which to use. """ @classmethod def setUpClass(cls): HelperTestMixin.setUpClass() def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient() self.IDS = IdentityManagementServiceClient(node=self.container.node) self.RR2 = EnhancedResourceRegistryClient(self.RR) self.org_id = self.RR2.create(any_old(RT.Org)) log.debug("Org created: %s", self.org_id) # Create InstrumentModel # TODO create multiple models as needed; for the moment assuming all # used instruments are the same model here. instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") self.instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', self.instModel_id) # Use the network definition provided by RSN OMS directly. rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri']) self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms) CIOMSClientFactory.destroy_instance(rsn_oms) if log.isEnabledFor(logging.TRACE): # show serialized version for the network definition: network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition) log.trace("NetworkDefinition serialization:\n%s", network_definition_ser) # set attributes for the platforms: self._platform_attributes = {} for platform_id in self._network_definition.pnodes: pnode = self._network_definition.pnodes[platform_id] dic = dict((attr.attr_id, attr.defn) for attr in pnode.attrs.itervalues()) self._platform_attributes[platform_id] = dic log.trace("_platform_attributes: %s", self._platform_attributes) # set ports for the platforms: self._platform_ports = {} for platform_id in self._network_definition.pnodes: pnode = self._network_definition.pnodes[platform_id] dic = {} for port_id, port in pnode.ports.iteritems(): dic[port_id] = dict(port_id=port_id, network=port.network) self._platform_ports[platform_id] = dic log.trace("_platform_ports: %s", self._platform_attributes) self._async_data_result = AsyncResult() self._data_subscribers = [] self._samples_received = [] self.addCleanup(self._stop_data_subscribers) self._async_event_result = AsyncResult() self._event_subscribers = [] self._events_received = [] self.addCleanup(self._stop_event_subscribers) self._start_event_subscriber(sub_type="platform_event") # instruments that have been set up: instr_key: i_obj self._setup_instruments = {} ################################################################# # data subscribers handling ################################################################# def _start_data_subscriber(self, stream_name, stream_id): """ Starts data subscriber for the given stream_name and stream_config """ def consume_data(message, stream_route, stream_id): # A callback for processing subscribed-to data. log.info('Subscriber received data message: %s. stream_name=%r stream_id=%r', str(message), stream_name, stream_id) self._samples_received.append(message) self._async_data_result.set() log.info('_start_data_subscriber stream_name=%r stream_id=%r', stream_name, stream_id) # Create subscription for the stream exchange_name = '%s_queue' % stream_name self.container.ex_manager.create_xn_queue(exchange_name).purge() sub = StandaloneStreamSubscriber(exchange_name, consume_data) sub.start() self._data_subscribers.append(sub) sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id]) self.PSC.activate_subscription(sub_id) sub.subscription_id = sub_id def _stop_data_subscribers(self): """ Stop the data subscribers on cleanup. """ try: for sub in self._data_subscribers: if hasattr(sub, 'subscription_id'): try: self.PSC.deactivate_subscription(sub.subscription_id) except: pass self.PSC.delete_subscription(sub.subscription_id) sub.stop() finally: self._data_subscribers = [] ################################################################# # event subscribers handling ################################################################# def _start_event_subscriber(self, event_type="DeviceEvent", sub_type=None, count=0): """ Starts event subscriber for events of given event_type ("DeviceEvent" by default) and given sub_type ("platform_event" by default). """ def consume_event(evt, *args, **kwargs): # A callback for consuming events. log.info('Event subscriber received evt: %s.', str(evt)) self._events_received.append(evt) if count == 0: self._async_event_result.set(evt) elif count == len(self._events_received): self._async_event_result.set() sub = EventSubscriber(event_type=event_type, sub_type=sub_type, callback=consume_event) sub.start() log.info("registered event subscriber for event_type=%r, sub_type=%r, count=%d", event_type, sub_type, count) self._event_subscribers.append(sub) sub._ready_event.wait(timeout=EVENT_TIMEOUT) def _stop_event_subscribers(self): """ Stops the event subscribers on cleanup. """ try: for sub in self._event_subscribers: if hasattr(sub, 'subscription_id'): try: self.PSC.deactivate_subscription(sub.subscription_id) except: pass self.PSC.delete_subscription(sub.subscription_id) sub.stop() finally: self._event_subscribers = [] ################################################################# # config supporting methods ################################################################# def _get_platform_stream_configs(self): """ This method is an adaptation of get_streamConfigs in test_driver_egg.py """ return [ StreamConfiguration(stream_name='parsed', parameter_dictionary_name='platform_eng_parsed', records_per_granule=2, granule_publish_rate=5) # TODO include a "raw" stream? ] def _get_instrument_stream_configs(self): """ configs copied from test_activate_instrument.py """ return [ StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5), StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5) ] def _verify_child_config(self, config, device_id, is_platform): for key in required_config_keys: self.assertIn(key, config) if is_platform: self.assertEqual(RT.PlatformDevice, config['device_type']) for key in DVR_CONFIG.iterkeys(): self.assertIn(key, config['driver_config']) for key in ['startup_config']: self.assertEqual({}, config[key]) else: self.assertEqual(RT.InstrumentDevice, config['device_type']) for key in ['children']: self.assertEqual({}, config[key]) self.assertEqual({'resource_id': device_id}, config['agent']) self.assertIn('stream_config', config) def _verify_parent_config(self, config, parent_device_id, child_device_id, is_platform): for key in required_config_keys: self.assertIn(key, config) self.assertEqual(RT.PlatformDevice, config['device_type']) for key in DVR_CONFIG.iterkeys(): self.assertIn(key, config['driver_config']) self.assertEqual({'resource_id': parent_device_id}, config['agent']) self.assertIn('stream_config', config) for key in ['startup_config']: self.assertEqual({}, config[key]) self.assertIn(child_device_id, config['children']) self._verify_child_config(config['children'][child_device_id], child_device_id, is_platform) def _create_platform_configuration(self, platform_id, parent_platform_id=None): """ This method is an adaptation of test_agent_instance_config in test_instrument_management_service_integration.py @param platform_id @param parent_platform_id @return a DotDict with various of the constructed elements associated to the platform. """ tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() # # TODO will each platform have its own param dictionary? # param_dict_name = 'platform_eng_parsed' parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name( param_dict_name, id_only=True) self.parsed_stream_def_id = self.PSC.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_rpdict_id) def _make_platform_agent_structure(agent_config=None): if None is agent_config: agent_config = {} driver_config = copy.deepcopy(DVR_CONFIG) driver_config['attributes'] = self._platform_attributes[platform_id] driver_config['ports'] = self._platform_ports[platform_id] log.debug("driver_config: %s", driver_config) # instance creation platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, { 'driver_config': driver_config}) platform_agent_instance_obj.agent_config = agent_config platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj) # agent creation platform_agent_obj = any_old(RT.PlatformAgent, { "stream_configurations": self._get_platform_stream_configs(), 'driver_module': DVR_MOD, 'driver_class': DVR_CLS}) platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj) # device creation platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice)) # data product creation dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom}) dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id) self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id) self.DP.activate_data_product_persistence(data_product_id=dp_id) self.addCleanup(self.DP.delete_data_product, dp_id) # assignments self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(platform_agent_instance_id, platform_device_id) self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, platform_agent_instance_id) self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, self.org_id) ####################################### # dataset log.debug('data product = %s', dp_id) stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, None, True) log.debug('Data product stream_ids = %s', stream_ids) stream_id = stream_ids[0] # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) ####################################### return platform_agent_instance_id, platform_agent_id, platform_device_id, stream_id log.debug("Making the structure for a platform agent") # TODO Note: the 'platform_config' entry is a mechanism that the # platform agent expects to know the platform_id and parent_platform_id. # Determine how to finally indicate this info. platform_config = { 'platform_id': platform_id, 'parent_platform_id': parent_platform_id, } child_agent_config = { 'platform_config': platform_config } platform_agent_instance_child_id, _, platform_device_child_id, stream_id = \ _make_platform_agent_structure(child_agent_config) platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id) self.platform_device_parent_id = platform_device_child_id p_obj = DotDict() p_obj.platform_id = platform_id p_obj.parent_platform_id = parent_platform_id p_obj.platform_agent_instance_obj = platform_agent_instance_child_obj p_obj.platform_device_id = platform_device_child_id p_obj.platform_agent_instance_id = platform_agent_instance_child_id p_obj.stream_id = stream_id p_obj.pid = None # known when process launched return p_obj def _create_platform(self, platform_id, parent_platform_id=None): """ The main method to create a platform configuration and do other preparations for a given platform. """ p_obj = self._create_platform_configuration(platform_id, parent_platform_id) # start corresponding data subscriber: self._start_data_subscriber(p_obj.platform_agent_instance_id, p_obj.stream_id) return p_obj ################################################################# # platform child-parent linking ################################################################# def _assign_child_to_parent(self, p_child, p_parent): log.debug("assigning child platform %r to parent %r", p_child.platform_id, p_parent.platform_id) self.RR2.assign_platform_device_to_platform_device_with_has_device(p_child.platform_device_id, p_parent.platform_device_id) child_device_ids = self.RR2.find_platform_device_ids_of_device_using_has_device(p_parent.platform_device_id) self.assertNotEqual(0, len(child_device_ids)) ################################################################# # instrument ################################################################# def _set_up_pre_environment_for_instrument(self, instr_info): """ Based on test_instrument_agent.py Basically, this method launches a port agent and then completes the instrument driver configuration used to properly set up a particular instrument agent. @param instr_info A value in instruments_dict @return instrument_driver_config """ import sys from ion.agents.instrument.driver_process import DriverProcessType from ion.agents.instrument.driver_process import ZMQEggDriverProcess # A seabird driver. DRV_URI = SBE37_EGG DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver' DRV_CLS = 'SBE37Driver' WORK_DIR = '/tmp/' DELIM = ['<<', '>>'] instrument_driver_config = { 'dvr_egg' : DRV_URI, 'dvr_mod' : DRV_MOD, 'dvr_cls' : DRV_CLS, 'workdir' : WORK_DIR, 'process_type' : None } # Launch from egg or a local MI repo. LAUNCH_FROM_EGG=True if LAUNCH_FROM_EGG: # Dynamically load the egg into the test path launcher = ZMQEggDriverProcess(instrument_driver_config) egg = launcher._get_egg(DRV_URI) if not egg in sys.path: sys.path.insert(0, egg) instrument_driver_config['process_type'] = (DriverProcessType.EGG,) else: mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo' if not mi_repo in sys.path: sys.path.insert(0, mi_repo) instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,) instrument_driver_config['mi_repo'] = mi_repo DEV_ADDR = instr_info['DEV_ADDR'] DEV_PORT = instr_info['DEV_PORT'] DATA_PORT = instr_info['DATA_PORT'] CMD_PORT = instr_info['CMD_PORT'] PA_BINARY = instr_info['PA_BINARY'] support = DriverIntegrationTestSupport(None, None, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = support.start_pagent() log.info('Port agent started at port %i', port) self.addCleanup(support.stop_pagent) # Configure instrument driver to use port agent port number. instrument_driver_config['comms_config'] = { 'addr': 'localhost', 'port': port, 'cmd_port': CMD_PORT } return instrument_driver_config def _make_instrument_agent_structure(self, instr_key, org_obj, agent_config=None): if None is agent_config: agent_config = {} instr_info = instruments_dict[instr_key] # initially adapted from test_activate_instrument:test_activateInstrumentSample # agent creation instrument_agent_obj = IonObject(RT.InstrumentAgent, name='agent007_%s' % instr_key, description="SBE37IMAgent_%s" % instr_key, driver_uri=SBE37_EGG, stream_configurations=self._get_instrument_stream_configs()) instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj) log.debug('new InstrumentAgent id = %s', instrument_agent_id) self.IMS.assign_instrument_model_to_instrument_agent(self.instModel_id, instrument_agent_id) # device creation instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice_%s' % instr_key, description="SBE37IMDevice_%s" % instr_key, serial_number="12345") instrument_device_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(self.instModel_id, instrument_device_id) log.debug("new InstrumentDevice id = %s ", instrument_device_id) #Create stream alarms temp_alert_def = { 'name' : 'temperature_warning_interval', 'stream_name' : 'parsed', 'message' : 'Temperature is below the normal range of 50.0 and above.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_DATA, 'value_id' : 'temp', 'lower_bound' : 50.0, 'lower_rel_op' : '<', 'alert_class' : 'IntervalAlert' } late_data_alert_def = { 'name' : 'late_data_warning', 'stream_name' : 'parsed', 'message' : 'Expected data has not arrived.', 'alert_type' : StreamAlertType.WARNING, 'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS, 'value_id' : None, 'time_delta' : 2, 'alert_class' : 'LateDataAlert' } instrument_driver_config = self._set_up_pre_environment_for_instrument(instr_info) port_agent_config = { 'device_addr': instr_info['DEV_ADDR'], 'device_port': instr_info['DEV_PORT'], 'data_port': instr_info['DATA_PORT'], 'command_port': instr_info['CMD_PORT'], 'binary_path': instr_info['PA_BINARY'], 'process_type': PortAgentProcessType.UNIX, 'port_agent_addr': 'localhost', 'log_level': 5, 'type': PortAgentType.ETHERNET } # instance creation instrument_agent_instance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance_%s' % instr_key, description="SBE37IMAgentInstance_%s" % instr_key, driver_config=instrument_driver_config, port_agent_config=port_agent_config, alerts=[temp_alert_def, late_data_alert_def]) instrument_agent_instance_obj.agent_config = agent_config instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj) # data products tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() org_id = self.RR2.create(org_obj) # parsed: parsed_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition( name='ctd_parsed', parameter_dictionary_id=parsed_pdict_id) dp_obj = IonObject(RT.DataProduct, name='the parsed data for %s' % instr_key, description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) self.addCleanup(self.DP.delete_data_product, data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) # raw: raw_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition( name='ctd_raw', parameter_dictionary_id=raw_pdict_id) dp_obj = IonObject(RT.DataProduct, name='the raw data for %s' % instr_key, description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) self.addCleanup(self.DP.delete_data_product, data_product_id2) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id2) # assignments self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance(instrument_agent_instance_id, instrument_device_id) self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition(instrument_agent_id, instrument_agent_instance_id) self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id) i_obj = DotDict() i_obj.instrument_agent_id = instrument_agent_id i_obj.instrument_device_id = instrument_device_id i_obj.instrument_agent_instance_id = instrument_agent_instance_id i_obj.org_obj = org_obj log.debug("KK CREATED I_obj: %s", i_obj) return i_obj def _create_instrument(self, instr_key): """ The main method to create an instrument configuration. @param instr_key A key in instruments_dict @return instrument_driver_config """ self.assertIn(instr_key, instruments_dict) self.assertNotIn(instr_key, self._setup_instruments) instr_info = instruments_dict[instr_key] log.debug("_create_instrument: creating instrument %r: %s", instr_key, instr_info) org_obj = any_old(RT.Org) log.debug("making the structure for an instrument agent") i_obj = self._make_instrument_agent_structure(instr_key, org_obj) self._setup_instruments[instr_key] = i_obj log.debug("_create_instrument: created instrument %r", instr_key) return i_obj def _get_instrument(self, instr_key): """ Gets the i_obj constructed by _create_instrument(instr_key). """ self.assertIn(instr_key, self._setup_instruments) i_obj = self._setup_instruments[instr_key] return i_obj ################################################################# # instrument-platform linking ################################################################# def _assign_instrument_to_platform(self, i_obj, p_obj): log.debug("assigning instrument %r to platform %r", i_obj.instrument_agent_instance_id, p_obj.platform_id) self.RR2.assign_instrument_device_to_platform_device_with_has_device( i_obj.instrument_device_id, p_obj.platform_device_id) child_device_ids = self.RR2.find_instrument_device_ids_of_device_using_has_device(p_obj.platform_device_id) self.assertNotEqual(0, len(child_device_ids)) ################################################################# # some platform topologies ################################################################# def _create_single_platform(self): """ Creates and prepares a platform corresponding to the platform ID 'LJ01D', which is a leaf in the simulated network. """ p_root = self._create_platform('LJ01D') return p_root def _create_small_hierarchy(self): """ Creates a small platform network consisting of 3 platforms as follows: Node1D -> MJ01C -> LJ01D where -> goes from parent to child. """ p_root = self._create_platform('Node1D') p_child = self._create_platform('MJ01C', parent_platform_id='Node1D') p_grandchild = self._create_platform('LJ01D', parent_platform_id='MJ01C') self._assign_child_to_parent(p_child, p_root) self._assign_child_to_parent(p_grandchild, p_child) return p_root def _create_hierarchy(self, platform_id, p_objs, parent_obj=None): """ Creates a hierarchy of platforms rooted at the given platform. @param platform_id ID of the root platform at this level @param p_objs dict to be updated with (platform_id: p_obj) mappings @param parent_obj platform object of the parent, if any @return platform object for the created root. """ # create the object to be returned: p_obj = self._create_platform(platform_id) # update (platform_id: p_obj) dict: p_objs[platform_id] = p_obj # recursively create child platforms: pnode = self._network_definition.pnodes[platform_id] for sub_platform_id in pnode.subplatforms: self._create_hierarchy(sub_platform_id, p_objs, p_obj) if parent_obj: self._assign_child_to_parent(p_obj, parent_obj) return p_obj def _set_up_single_platform_with_some_instruments(self, instr_keys): """ Sets up single platform with some instruments @param instr_keys Keys of the instruments to be assigned. Must be keys in instruments_dict in base_test_platform_agent_with_rsn @return p_root for subsequent termination """ for instr_key in instr_keys: self.assertIn(instr_key, instruments_dict) p_root = self._create_single_platform() # create and assign instruments: for instr_key in instr_keys: i_obj = self._create_instrument(instr_key) self._assign_instrument_to_platform(i_obj, p_root) return p_root def _set_up_platform_hierarchy_with_some_instruments(self, instr_keys): """ Sets up a multiple-level platform hierarchy with instruments associated to some of the platforms. The platform hierarchy corresponds to the sub-network in the simulated topology rooted at 'Node1B', which at time of writing looks like this: Node1B Node1C Node1D MJ01C LJ01D LV01C PC01B SC01B SF01B LJ01C LV01B LJ01B MJ01B In DEBUG logging level for the platform agent, files like the following are generated under logs/: platform_CFG_received_Node1B.txt platform_CFG_received_MJ01C.txt platform_CFG_received_LJ01D.txt @param instr_keys Keys of the instruments to be assigned. Must be keys in instruments_dict in base_test_platform_agent_with_rsn @return p_root for subsequent termination """ for instr_key in instr_keys: self.assertIn(instr_key, instruments_dict) ##################################### # create platform hierarchy ##################################### log.info("will create platform hierarchy ...") start_time = time.time() root_platform_id = 'Node1B' p_objs = {} p_root = self._create_hierarchy(root_platform_id, p_objs) log.info("platform hierarchy built. Took %.3f secs. " "Root platform=%r, number of platforms=%d: %s", time.time() - start_time, root_platform_id, len(p_objs), p_objs.keys()) self.assertIn(root_platform_id, p_objs) self.assertEquals(13, len(p_objs)) ##################################### # create the indicated instruments ##################################### log.info("will create %d instruments: %s", len(instr_keys), instr_keys) start_time = time.time() i_objs = [] for instr_key in instr_keys: i_obj = self._create_instrument(instr_key) i_objs.append(i_obj) log.debug("instrument created = %r (%s)", i_obj.instrument_agent_instance_id, instr_key) log.info("%d instruments created. Took %.3f secs.", len(instr_keys), time.time() - start_time) ##################################### # assign the instruments ##################################### log.info("will assign instruments ...") start_time = time.time() plats_to_assign_instrs = [ 'LJ01D', 'SF01B', 'LJ01B', 'MJ01B', # leaves 'MJ01C', 'Node1D', 'LV01B', 'Node1C' # intermediate ] # assign one available instrument to a platform; # the assignments are arbitrary. num_assigns = min(len(instr_keys), len(plats_to_assign_instrs)) for ii in range(num_assigns): platform_id = plats_to_assign_instrs[ii] self.assertIn(platform_id, p_objs) p_obj = p_objs[platform_id] i_obj = i_objs[ii] self._assign_instrument_to_platform(i_obj, p_obj) log.debug("instrument %r (%s) assigned to platform %r", i_obj.instrument_agent_instance_id, instr_keys[ii], platform_id) log.info("%d instruments assigned. Took %.3f secs.", num_assigns, time.time() - start_time) return p_root ################################################################# # start / stop platform ################################################################# def _start_platform(self, p_obj): agent_instance_id = p_obj.platform_agent_instance_id log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id) p_obj.pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id) log.debug("start_platform_agent_instance returned pid=%s", p_obj.pid) #wait for start agent_instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id) gate = ProcessStateGate(self.PDC.read_process, agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds") # Start a resource agent client to talk with the agent. self._pa_client = ResourceAgentClient('paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("got platform agent client %s", str(self._pa_client)) def _stop_platform(self, p_obj): try: self.IMS.stop_platform_agent_instance(p_obj.platform_agent_instance_id) except: if log.isEnabledFor(logging.TRACE): log.exception( "platform_id=%r: Exception in IMS.stop_platform_agent_instance with " "platform_agent_instance_id = %r", p_obj.platform_id, p_obj.platform_agent_instance_id) else: log.warn( "platform_id=%r: Exception in IMS.stop_platform_agent_instance with " "platform_agent_instance_id = %r. Perhaps already dead.", p_obj.platform_id, p_obj.platform_agent_instance_id) ################################################################# # misc convenience methods ################################################################# def _create_resource_agent_client(self, resource_id): client = ResourceAgentClient(resource_id, process=FakeProcess()) return client def _get_state(self): state = self._pa_client.get_agent_state() return state def _assert_state(self, state): self.assertEquals(self._get_state(), state) def _execute_agent(self, cmd): log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs) time_start = time.time() #retval = self._pa_client.execute_agent(cmd, timeout=timeout) retval = self._pa_client.execute_agent(cmd) elapsed_time = time.time() - time_start log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s", cmd.command, elapsed_time, str(retval)) return retval ################################################################# # commands that concrete tests can call ################################################################# def _ping_agent(self): retval = self._pa_client.ping_agent() self.assertIsInstance(retval, str) def _ping_resource(self): cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE) if self._get_state() == PlatformAgentState.UNINITIALIZED: # should get ServerError: "Command not handled in current state" with self.assertRaises(ServerError): self._pa_client.execute_agent(cmd) else: # In all other states the command should be accepted: retval = self._execute_agent(cmd) self.assertEquals("PONG", retval.result) def _get_metadata(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_METADATA = %s", md) def _get_ports(self): cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS) retval = self._execute_agent(cmd) md = retval.result self.assertIsInstance(md, dict) # TODO verify possible subset of required entries in the dict. log.info("GET_PORTS = %s", md) def _initialize(self): self._assert_state(PlatformAgentState.UNINITIALIZED) cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _go_active(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.IDLE) def _run(self): cmd = AgentCommand(command=PlatformAgentEvent.RUN) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _start_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.MONITORING) def _wait_for_a_data_sample(self): log.info("waiting for reception of a data sample...") # just wait for at least one -- see consume_data self._async_data_result.get(timeout=DATA_TIMEOUT) self.assertTrue(len(self._samples_received) >= 1) log.info("Received samples: %s", len(self._samples_received)) def _wait_for_external_event(self): log.info("waiting for reception of an external event...") # just wait for at least one -- see consume_event self._async_event_result.get(timeout=EVENT_TIMEOUT) self.assertTrue(len(self._events_received) >= 1) log.info("Received events: %s", len(self._events_received)) def _stop_resource_monitoring(self): cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _pause(self): cmd = AgentCommand(command=PlatformAgentEvent.PAUSE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.STOPPED) def _resume(self): cmd = AgentCommand(command=PlatformAgentEvent.RESUME) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.COMMAND) def _clear(self): cmd = AgentCommand(command=PlatformAgentEvent.CLEAR) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.IDLE) def _go_inactive(self): cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.INACTIVE) def _reset(self): cmd = AgentCommand(command=PlatformAgentEvent.RESET) retval = self._execute_agent(cmd) self._assert_state(PlatformAgentState.UNINITIALIZED) def _check_sync(self): cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC) retval = self._execute_agent(cmd) log.info("CHECK_SYNC result: %s", retval.result) self.assertTrue(retval.result is not None) self.assertEquals(retval.result[0:3], "OK:") return retval.result def _stream_instruments(self): from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37ProtocolEvent from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37Parameter for instrument in self._setup_instruments.itervalues(): # instruments that have been set up: instr_key: i_obj # Start a resource agent client to talk with the instrument agent. _ia_client = self._create_resource_agent_client(instrument.instrument_device_id) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = _ia_client.execute_resource(cmd) log.debug('_stream_instruments retval: %s', retval) return def _idle_instruments(self): from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37ProtocolEvent from mi.instrument.seabird.sbe37smb.ooicore.driver import SBE37Parameter for instrument in self._setup_instruments.itervalues(): # instruments that have been set up: instr_key: i_obj # Start a resource agent client to talk with the instrument agent. _ia_client = self._create_resource_agent_client(instrument.instrument_device_id) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) with self.assertRaises(Conflict): retval = _ia_client.execute_resource(cmd) cmd = AgentCommand(command=ResourceAgentEvent.RESET) retval = _ia_client.execute_agent(cmd) state = _ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) return
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container super(TestActivateInstrumentIntegration, self).setUp() config = DotDict() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml', config) # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() self.usernotificationclient = UserNotificationServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration) return pid def _create_notification(self, user_name = '', instrument_id='', product_id=''): #-------------------------------------------------------------------------------------- # Make notification request objects #-------------------------------------------------------------------------------------- notification_request_1 = NotificationRequest( name= 'notification_1', origin=instrument_id, origin_type="instrument", event_type='ResourceLifecycleEvent') notification_request_2 = NotificationRequest( name='notification_2', origin=product_id, origin_type="data product", event_type='DetectionEvent') #-------------------------------------------------------------------------------------- # Create a user and get the user_id #-------------------------------------------------------------------------------------- user = UserInfo() user.name = user_name user.contact.email = '*****@*****.**' % user_name user_id, _ = self.rrclient.create(user) #-------------------------------------------------------------------------------------- # Create notification #-------------------------------------------------------------------------------------- self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id) self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id) log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) ) return user_id def get_datastore(self, dataset_id): dataset = self.datasetclient.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None): # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue) self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue) # the following assert will not work without elasticsearch. #self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) # Verify the computed attribute for user notification requests self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) ) notifications = extended_instrument.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_instrument_device_id, notification.origin) self.assertEqual("instrument", notification.origin_type) self.assertEqual('ResourceLifecycleEvent', notification.event_type) def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None): self.assertEqual(expected_data_product_id, extended_data_product._id) log.debug("extended_data_product.computed: %s", extended_data_product.computed) # Verify that computed attributes exist for the extended instrument self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedFloatValue) self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue) self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue) self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue) self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue) self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue) self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue) self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue) self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue) # exact text here keeps changing to fit UI capabilities. keep assertion general... self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) ) notifications = extended_data_product.computed.user_notification_requests.value notification = notifications[0] self.assertEqual(expected_data_product_id, notification.origin) self.assertEqual("data product", notification.origin_type) self.assertEqual('DetectionEvent', notification.event_type) @attr('LOCOINT') #@unittest.skip('refactoring') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 90}}}) def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw') parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config, alerts= []) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s' , data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s' , dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s' , str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s' , dataset_ids[0]) self.raw_dataset = dataset_ids[0] def start_instrument_agent(): self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % gate.process_id) #log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s" , str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.INACTIVE, state) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s" , str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.STOPPED, state) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.IDLE, state) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(ResourceAgentState.COMMAND, state) for i in xrange(10): monitor = DatasetMonitor(dataset_id=self.parsed_dataset) self._ia_client.execute_resource(AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)) if not monitor.wait(): raise AssertionError('Failed on the %ith granule' % i) monitor.stop() # cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) # for i in xrange(10): # retval = self._ia_client.execute_resource(cmd) # log.debug("test_activateInstrumentSample: return from sample %s" , str(retval)) log.debug( "test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s" , str(reply)) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data_raw, Granule) rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw) log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) ) self.assertIn('raw', rdt_raw) raw_vals = rdt_raw['raw'] all_raw = "".join(raw_vals) # look for 't' entered after a prompt -- ">t" t_commands = all_raw.count(">t") if 10 != t_commands: log.error("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.error("raw val %s: %s", i, [r]) self.fail("Expected 10 't' strings in raw_vals, got %s" % t_commands) else: log.debug("%s raw_vals: ", len(raw_vals)) for i, r in enumerate(raw_vals): log.debug("raw val %s: %s", i, [r]) replay_data_parsed = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data_parsed, Granule) rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt_parsed.pretty_print()) ) self.assertIn('temp', rdt_parsed) temp_vals = rdt_parsed['temp'] pressure_vals = rdt_parsed['pressure'] if 10 != len(temp_vals): log.error("%s temp_vals: %s", len(temp_vals), temp_vals) self.fail("Expected 10 temp_vals, got %s" % len(temp_vals)) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests.value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.IDS = IdentityManagementServiceClient(node=self.container.node) self.PSC = PubsubManagementServiceClient(node=self.container.node) self.DP = DataProductManagementServiceClient(node=self.container.node) self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node) self.DSC = DatasetManagementServiceClient(node=self.container.node) self.PDC = ProcessDispatcherServiceClient(node=self.container.node) print 'started services' # @unittest.skip('this test just for debugging setup') # def test_just_the_setup(self): # return @attr('EXT') def test_resources_associations_extensions(self): """ create one of each resource and association used by IMS to guard against problems in ion-definitions """ #stuff we control instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance)) instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent)) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice)) platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance)) platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent)) platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice)) sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel)) #stuff we associate to data_producer_id, _ = self.RR.create(any_old(RT.DataProducer)) org_id, _ = self.RR.create(any_old(RT.Org)) #instrument_agent_instance_id #is only a target #instrument_agent self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id) self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id) self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id) self.RR.create_association(org_id, PRED.hasResource, instrument_device_id) instrument_model_id #is only a target platform_agent_instance_id #is only a target #platform_agent self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id) self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id) platform_model_id #is only a target #sensor_device self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id) self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id) sensor_model_id #is only a target #create a parsed product for this instrument output tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', processing_level_code='Parsed_Canonical', temporal_domain = tdom, spatial_domain = sdom) pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1) def addInstOwner(inst_id, subject): actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject}) user_id = self.IDS.create_actor_identity(actor_identity_obj) user_info_obj = any_old(RT.UserInfo) user_info_id = self.IDS.create_user_info(user_id, user_info_obj) self.RR.create_association(inst_id, PRED.hasOwner, user_id) #Testing multiple instrument owners addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254") addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256") extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id) self.assertEqual(instrument_device_id, extended_instrument._id) self.assertEqual(len(extended_instrument.owners), 2) self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id) #check model inst_model_obj = self.RR.read(instrument_model_id) self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name) #check agent instance inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id) self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name) #check agent inst_agent_obj = self.RR.read(instrument_agent_id) #compound assoc return list of lists so check the first element self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name) #check platform device plat_device_obj = self.RR.read(platform_device_id) self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name) #check sensor devices self.assertEqual(1, len(extended_instrument.sensor_devices)) #check data_product_parameters_set self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_product_parameters_set.status) self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value) # the ctd parameters should include 'temp' self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical']) #none of these will work because there is no agent self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.firmware_version.status) self.assertEqual(ComputedValueAvailability.NOTAVAILABLE, extended_instrument.computed.operational_state.status) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.power_status_roll_up.status) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.communications_status_roll_up.status) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.data_status_roll_up.status) self.assertEqual(StatusType.STATUS_OK, extended_instrument.computed.data_status_roll_up.value) self.assertEqual(ComputedValueAvailability.PROVIDED, extended_instrument.computed.location_status_roll_up.status) # self.assertEqual(ComputedValueAvailability.PROVIDED, # extended_instrument.computed.recent_events.status) # self.assertEqual([], extended_instrument.computed.recent_events.value) # cleanup c = DotDict() c.resource_registry = self.RR resource_impl = ResourceImpl(c) resource_impl.pluck(instrument_agent_id) resource_impl.pluck(instrument_model_id) resource_impl.pluck(instrument_device_id) resource_impl.pluck(platform_agent_id) self.IMS.force_delete_instrument_agent(instrument_agent_id) self.IMS.force_delete_instrument_model(instrument_model_id) self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id) self.IMS.force_delete_platform_agent(platform_agent_id) self.IMS.force_delete_platform_device(platform_device_id) self.IMS.force_delete_platform_model(platform_model_id) self.IMS.force_delete_sensor_device(sensor_device_id) self.IMS.force_delete_sensor_model(sensor_model_id) #stuff we associate to self.RR.delete(data_producer_id) self.RR.delete(org_id) def test_custom_attributes(self): """ Test assignment of custom attributes """ instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel, {"custom_attributes": {"favorite_color": "attr desc goes here"} })) instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice, {"custom_attributes": {"favorite_color": "red", "bogus_attr": "should raise warning" } })) self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id) # cleanup self.IMS.force_delete_instrument_device(instrument_device_id) self.IMS.force_delete_instrument_model(instrument_model_id) def _get_datastore(self, dataset_id): dataset = self.DSC.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def test_checkpoint_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", stream_configuration= {'raw': 'ctd_raw_param_dict' , 'parsed': 'ctd_parsed_param_dict' }) instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict') parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict') raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id, parameter_dictionary=spdict_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data self._get_datastore(self.parsed_dataset) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id, parameter_dictionary=rpdict_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.agent_state_checkpoint(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.agent_state_restore(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
class TestDriverEgg(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] self.event_publisher = EventPublisher() def get_streamConfigs(self): raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') return raw_config, parsed_config ########################## # # The following tests generate different agent configs and pass them to a common base test script # ########################### @unittest.skip("this test can't be run from coi services. it is missing dependencies") def test_driverLaunchModuleNoURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchNoModuleOnlyURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) def test_driverLaunchBogusModuleWithURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="bogus", driver_class="Bogus", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj) @unittest.skip("Launches an egg 'process' even though the egg download should produce error 404") def test_driverLaunchNoModule404URI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_404, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, False) def test_driverLaunchNoModuleBadEggURI(self): raw_config, parsed_config = self.get_streamConfigs() instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", #driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", #driver_class="SBE37Driver", driver_uri=DRV_URI_BAD, stream_configurations = [raw_config, parsed_config]) self.base_activateInstrumentSample(instAgent_obj, True, False) def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True): """ This method runs a test of launching a driver with a given agent configuration """ # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) print 'new InstrumentModel id = %s ' % instModel_id # Create InstrumentAgent instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) print 'new InstrumentAgent id = %s' % instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new dp_id = %s' % data_product_id1 self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) print 'Data product streams1 = %s' % stream_ids # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id1 = %s' % dataset_ids[0] self.parsed_dataset = dataset_ids[0] dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) print 'new dp_id = %s' % str(data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) print 'Data product streams2 = %s' % str(stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id2 = %s' % dataset_ids[0] self.raw_dataset = dataset_ids[0] # add start/stop for instrument agent gevent.joinall([gevent.spawn(lambda: self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id))]) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id) agent_process_id = ResourceAgentClient._get_agent_process_id(instDevice_id) print "Agent process id is '%s'" % str(agent_process_id) self.assertTrue(agent_process_id) gate = ProcessStateGate(self.processdispatchclient.read_process, agent_process_id, ProcessStateEnum.RUNNING) if not expect_launch: self.assertFalse(gate.await(30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" % agent_process_id) return self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % agent_process_id) print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING" #print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=agent_process_id, process=FakeProcess()) print "ResourceAgentClient created: %s" % str(self._ia_client) print "Sending command=ResourceAgentEvent.INITIALIZE" cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) if not expect_command: self.assertRaises(ServerError, self._ia_client.execute_agent, cmd) return retval = self._ia_client.execute_agent(cmd) print "Result of INITIALIZE: %s" % str(retval) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) print "Sending command=ResourceAgentEvent.RESET" cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) print "Result of RESET: %s" % str(reply)
class TestCTDTransformsIntegration(IonIntegrationTestCase): pdict_id = None def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) self.dataset_management = self.datasetclient def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration) return pid def _create_instrument_model(self): instModel_obj = IonObject( RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) return instModel_id def _create_instrument_agent(self, instModel_id): raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict') parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict') instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) return instAgent_id def _create_instrument_device(self, instModel_id): instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) return instDevice_id def _create_instrument_agent_instance(self, instAgent_id,instDevice_id): port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) return instAgentInstance_id def _create_param_dicts(self): tdom, sdom = time_series_domain() self.sdom = sdom.dump() self.tdom = tdom.dump() self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) def _create_input_data_products(self, ctd_stream_def_id, instDevice_id, ): dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = self.tdom, spatial_domain = self.sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) #--------------------------------------------------------------------------- # Create CTD Raw as the second data product #--------------------------------------------------------------------------- if not self.pdict_id: self._create_param_dicts() raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=self.pdict_id) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = self.tdom, spatial_domain = self.sdom) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #--------------------------------------------------------------------------- # Retrieve the id of the OUTPUT stream from the out Data Product #--------------------------------------------------------------------------- stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids return ctd_parsed_data_product def _create_data_process_definitions(self): #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L1 Conductivity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L1 Pressure: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L1 Temperature: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L2 Salinity: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------------------------------------------------------------- # L2 Density: Data Process Definition #------------------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) return self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id,\ self.ctd_L1_pressure_dprocdef_id,self.ctd_L1_temperature_dprocdef_id, \ self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id def _create_stream_definitions(self): if not self.pdict_id: self._create_param_dicts() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding= 'pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding='temperature' ) return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id def _create_l0_output_data_products(self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id): out_data_prods = [] ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) out_data_prods.append(self.ctd_l0_conductivity_output_dp_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_conductivity_output_dp_id) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) out_data_prods.append(self.ctd_l0_pressure_output_dp_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_pressure_output_dp_id) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) out_data_prods.append(self.ctd_l0_temperature_output_dp_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_temperature_output_dp_id) return out_data_prods def _create_l1_out_data_products(self): ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] ) self.loggerpids.append(pid) ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0] ) self.loggerpids.append(pid) ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0] ) self.loggerpids.append(pid) def _create_l2_out_data_products(self): #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- if not self.pdict_id: self._create_param_dicts() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding='density' ) ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0] ) self.loggerpids.append(pid) ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain = self.tdom, spatial_domain = self.sdom) self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0] ) self.loggerpids.append(pid) @unittest.skip('This test errors on coi-nightly, may be OBE.') def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] #------------------------------------------------------------------------------------- # Create InstrumentModel #------------------------------------------------------------------------------------- instModel_id = self._create_instrument_model() #------------------------------------------------------------------------------------- # Create InstrumentAgent #------------------------------------------------------------------------------------- instAgent_id = self._create_instrument_agent(instModel_id) #------------------------------------------------------------------------------------- # Create InstrumentDevice #------------------------------------------------------------------------------------- instDevice_id = self._create_instrument_device(instModel_id) #------------------------------------------------------------------------------------- # Create Instrument Agent Instance #------------------------------------------------------------------------------------- instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id,instDevice_id ) #------------------------------------------------------------------------------------- # create a stream definition for the data from the ctd simulator #------------------------------------------------------------------------------------- self._create_param_dicts() ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=self.pdict_id) #------------------------------------------------------------------------------------- # Create two data products #------------------------------------------------------------------------------------- ctd_parsed_data_product = self._create_input_data_products(ctd_stream_def_id,instDevice_id) #------------------------------------------------------------------------------------- # Create data process definitions #------------------------------------------------------------------------------------- self._create_data_process_definitions() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------------------------------------------------------------- outgoing_stream_l0_conductivity_id, \ outgoing_stream_l0_pressure_id, \ outgoing_stream_l0_temperature_id = self._create_stream_definitions() self.out_prod_ids = self._create_l0_output_data_products(outgoing_stream_l0_conductivity_id,outgoing_stream_l0_pressure_id,outgoing_stream_l0_temperature_id) self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding='pressure' ) self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding= 'temperature' ) self._create_l1_out_data_products() self._create_l2_out_data_products() #------------------------------------------------------------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------------------------------------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = self.ctd_L0_all_dprocdef_id, in_data_product_ids = [ctd_parsed_data_product], out_data_product_ids = self.out_prod_ids) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process,process_ids[0]) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEquals(extended_process.computed.operational_state.status, ComputedValueAvailability.NOTAVAILABLE) self.assertEquals(data_process.message_controllable, True) #------------------------------------------------------------------------------------- # L1 Conductivity: Create the data process #------------------------------------------------------------------------------------- l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_conductivity_dprocdef_id, [self.ctd_l0_conductivity_output_dp_id], [self.ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) data_process = self.rrclient.read(l1_conductivity_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l1_conductivity_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process,process_ids[0]) #------------------------------------------------------------------------------------- # L1 Pressure: Create the data process #------------------------------------------------------------------------------------- l1_pressure_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_pressure_dprocdef_id, [self.ctd_l0_pressure_output_dp_id], [self.ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) data_process = self.rrclient.read(l1_pressure_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l1_pressure_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L1 Temperature: Create the data process #------------------------------------------------------------------------------------- l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_temperature_dprocdef_id, [self.ctd_l0_temperature_output_dp_id], [self.ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) data_process = self.rrclient.read(l1_temperature_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l1_temperature_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Salinity: Create the data process #------------------------------------------------------------------------------------- l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) data_process = self.rrclient.read(l2_salinity_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l2_salinity_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # L2 Density: Create the data process #------------------------------------------------------------------------------------- l2_density_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_density_output_dp_id]) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) data_process = self.rrclient.read(l2_density_all_data_process_id) process_ids, _ = self.rrclient.find_objects(subject=l2_density_all_data_process_id, predicate=PRED.hasProcess, id_only=True) self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0]) #------------------------------------------------------------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------------------------------------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) # Wait for instrument agent to spawn gate = AgentProcessStateGate(self.processdispatchclient.read_process, instDevice_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds") # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess()) #------------------------------------------------------------------------------------- # Streaming #------------------------------------------------------------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #todo ResourceAgentClient no longer has method set_param # # Make sure the sampling rate and transmission are sane. # params = { # SBE37Parameter.NAVG : 1, # SBE37Parameter.INTERVAL : 5, # SBE37Parameter.TXREALTIME : True # } # self._ia_client.set_param(params) #todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it? cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) #todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it? # log.debug("test_activateInstrumentStream: calling go_observatory") # cmd = AgentCommand(command='go_observatory') # reply = self._ia_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._ia_client.execute_agent(cmd) # state = retval.result # log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) #------------------------------------------------------------------------------------------------- # Cleanup processes #------------------------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) #-------------------------------------------------------------------------------- # Cleanup data products #-------------------------------------------------------------------------------- dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct, id_only=True) for dp_id in dp_ids: self.dataproductclient.delete_data_product(dp_id)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.dataset_management = self.datasetclient def test_createDataProcess(self): #--------------------------------------------------------------------------- # Data Process Definition #--------------------------------------------------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) # Make assertion on the newly registered data process definition data_process_definition = self.rrclient.read(dprocdef_id) self.assertEquals(data_process_definition.name, 'ctd_L0_all') self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams') self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all') self.assertEquals(data_process_definition.class_name, 'ctd_L0_all') # Read the data process definition using data process management and make assertions dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id) self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all') self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all') #--------------------------------------------------------------------------- # Create an input instrument #--------------------------------------------------------------------------- instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) # Register the instrument so that the data producer and stream object are created data_producer_id = self.damsclient.register_instrument(instrument_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id ) # Assert that the link between the stream definition and the data process definition was done assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True) self.assertIsNotNone(assocs) #--------------------------------------------------------------------------- # Input Data Product #--------------------------------------------------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() input_dp_obj = IonObject( RT.DataProduct, name='InputDataProduct', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test') #Make assertions on the input data product created input_dp_obj = self.rrclient.read(input_dp_id) self.assertEquals(input_dp_obj.name, 'InputDataProduct') self.assertEquals(input_dp_obj.description, 'some new dp') self.damsclient.assign_data_product(instrument_id, input_dp_id) # Retrieve the stream via the DataProduct->Stream associations stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True) self.in_stream_id = stream_ids[0] #--------------------------------------------------------------------------- # Output Data Product #--------------------------------------------------------------------------- outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' ) outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' ) outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' ) self.output_products={} output_dp_obj = IonObject(RT.DataProduct, name='conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id) self.output_products['conductivity'] = output_dp_id_1 output_dp_obj = IonObject(RT.DataProduct, name='pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id) self.output_products['pressure'] = output_dp_id_2 output_dp_obj = IonObject(RT.DataProduct, name='temperature', description='transform output ', temporal_domain = tdom, spatial_domain = sdom) output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id) self.output_products['temperature'] = output_dp_id_3 #--------------------------------------------------------------------------- # Create the data process #--------------------------------------------------------------------------- def _create_data_process(): dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products) return dproc_id dproc_id = _create_data_process() # Make assertions on the data process created data_process = self.dataprocessclient.read_data_process(dproc_id) # Assert that the data process has a process id attached self.assertIsNotNone(data_process.process_id) # Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute self.assertIsNotNone(data_process.input_subscription_id) output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True) self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3])) @patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}}) def test_createDataProcessUsingSim(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" ) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", svr_addr="localhost", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) #------------------------------- # Create CTD Raw as the second data product #------------------------------- raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj.name = 'ctd_raw' ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # Create listener for data process events and verify that events are received. #------------------------------- # todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events #todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) #------------------------------- # Retrieve a list of all data process defintions in RR and validate that the DPD is listed #------------------------------- # todo: add this validate for Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions # todo: This capability is not yet completed (Swarbhanu) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) #todo: check that activate event is received L4-CI-SA-RQ-367 #todo... (it looks like no event is being published when the data process is activated... so below, we just check for now # todo... that the subscription is indeed activated) (Swarbhanu) # todo: monitor process to see if it is active (sa-rq-182) ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id) input_subscription_id = ctd_l0_all_data_process.input_subscription_id subs = self.rrclient.read(input_subscription_id) self.assertTrue(subs.activated) # todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the extended resources for data process definition and for data process #------------------------------- extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) self.assertEqual(1, len(extended_process_definition.data_processes)) log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition)) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEqual(1, len(extended_process.input_data_products)) log.debug("test_createDataProcess: extended_process %s", str(extended_process)) #------------------------------- # Cleanup #------------------------------- self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id) self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id) self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id) self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id)
class TestObservatoryManagementServiceIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.RR = ResourceRegistryServiceClient(node=self.container.node) self.OMS = ObservatoryManagementServiceClient(node=self.container.node) self.org_management_service = OrgManagementServiceClient(node=self.container.node) self.IMS = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() #print 'TestObservatoryManagementServiceIntegration: started services' self.event_publisher = EventPublisher() # @unittest.skip('this exists only for debugging the launch process') # def test_just_the_setup(self): # return def destroy(self, resource_ids): self.OMS.force_delete_observatory(resource_ids.observatory_id) self.OMS.force_delete_subsite(resource_ids.subsite_id) self.OMS.force_delete_subsite(resource_ids.subsite2_id) self.OMS.force_delete_subsite(resource_ids.subsiteb_id) self.OMS.force_delete_subsite(resource_ids.subsitez_id) self.OMS.force_delete_platform_site(resource_ids.platform_site_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb_id) self.OMS.force_delete_platform_site(resource_ids.platform_siteb2_id) self.OMS.force_delete_platform_site(resource_ids.platform_site3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site2_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_siteb3_id) self.OMS.force_delete_instrument_site(resource_ids.instrument_site4_id) #@unittest.skip('targeting') def test_resources_associations(self): resources = self._make_associations() self.destroy(resources) #@unittest.skip('targeting') def test_find_related_frames_of_reference(self): # finding subordinates gives a dict of obj lists, convert objs to ids def idify(adict): ids = {} for k, v in adict.iteritems(): ids[k] = [] for obj in v: ids[k].append(obj._id) return ids # a short version of the function we're testing, with id-ify def short(resource_id, output_types): ret = self.OMS.find_related_frames_of_reference(resource_id, output_types) return idify(ret) #set up associations first stuff = self._make_associations() #basic traversal of tree from instrument to platform ids = short(stuff.instrument_site_id, [RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) #since this is the first search, just make sure the input inst_id got stripped if RT.InstrumentSite in ids: self.assertNotIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #basic traversal of tree from platform to instrument ids = short(stuff.platform_siteb_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) self.assertNotIn(stuff.instrument_site2_id, ids[RT.InstrumentSite]) #full traversal of tree from observatory down to instrument ids = short(stuff.observatory_id, [RT.InstrumentSite]) self.assertIn(RT.InstrumentSite, ids) self.assertIn(stuff.instrument_site_id, ids[RT.InstrumentSite]) #full traversal of tree from instrument to observatory ids = short(stuff.instrument_site_id, [RT.Observatory]) self.assertIn(RT.Observatory, ids) self.assertIn(stuff.observatory_id, ids[RT.Observatory]) #partial traversal, only down to platform ids = short(stuff.observatory_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site3_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(RT.InstrumentSite, ids) #partial traversal, only down to platform ids = short(stuff.instrument_site_id, [RT.Subsite, RT.PlatformSite]) self.assertIn(RT.PlatformSite, ids) self.assertIn(RT.Subsite, ids) self.assertIn(stuff.platform_siteb_id, ids[RT.PlatformSite]) self.assertIn(stuff.platform_site_id, ids[RT.PlatformSite]) self.assertIn(stuff.subsite_id, ids[RT.Subsite]) self.assertIn(stuff.subsiteb_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsite2_id, ids[RT.Subsite]) self.assertNotIn(stuff.subsitez_id, ids[RT.Subsite]) self.assertNotIn(stuff.platform_siteb2_id, ids[RT.PlatformSite]) self.assertNotIn(RT.Observatory, ids) self.destroy(stuff) def _make_associations(self): """ create one of each resource and association used by OMS to guard against problems in ion-definitions """ #raise unittest.SkipTest("https://jira.oceanobservatories.org/tasks/browse/CISWCORE-41") """ the tree we're creating (observatory, sites, platforms, instruments) rows are lettered, colums numbered. - first row is implied a - first column is implied 1 - site Z, just because O--Sz | S--S2--P3--I4 | Sb-Pb2-Ib3 | P--I2 <- PlatformDevice, InstrumentDevice2 | Pb <- PlatformDevice b | I <- InstrumentDevice """ org_id = self.OMS.create_marine_facility(any_old(RT.Org)) def create_under_org(resource_type): obj = any_old(resource_type) if RT.InstrumentDevice == resource_type: resource_id = self.IMS.create_instrument_device(obj) else: resource_id, _ = self.RR.create(obj) self.OMS.assign_resource_to_observatory_org(resource_id=resource_id, org_id=org_id) return resource_id #stuff we control observatory_id = create_under_org(RT.Observatory) subsite_id = create_under_org(RT.Subsite) subsite2_id = create_under_org(RT.Subsite) subsiteb_id = create_under_org(RT.Subsite) subsitez_id = create_under_org(RT.Subsite) platform_site_id = create_under_org(RT.PlatformSite) platform_siteb_id = create_under_org(RT.PlatformSite) platform_siteb2_id = create_under_org(RT.PlatformSite) platform_site3_id = create_under_org(RT.PlatformSite) instrument_site_id = create_under_org(RT.InstrumentSite) instrument_site2_id = create_under_org(RT.InstrumentSite) instrument_siteb3_id = create_under_org(RT.InstrumentSite) instrument_site4_id = create_under_org(RT.InstrumentSite) #stuff we associate to instrument_device_id = create_under_org(RT.InstrumentDevice) instrument_device2_id = create_under_org(RT.InstrumentDevice) platform_device_id = create_under_org(RT.PlatformDevice) platform_deviceb_id = create_under_org(RT.PlatformDevice) instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel)) platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel)) deployment_id, _ = self.RR.create(any_old(RT.Deployment)) #observatory self.RR.create_association(observatory_id, PRED.hasSite, subsite_id) self.RR.create_association(observatory_id, PRED.hasSite, subsitez_id) #site self.RR.create_association(subsite_id, PRED.hasSite, subsite2_id) self.RR.create_association(subsite_id, PRED.hasSite, subsiteb_id) self.RR.create_association(subsite2_id, PRED.hasSite, platform_site3_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_siteb2_id) self.RR.create_association(subsiteb_id, PRED.hasSite, platform_site_id) #platform_site(s) self.RR.create_association(platform_site3_id, PRED.hasSite, instrument_site4_id) self.RR.create_association(platform_siteb2_id, PRED.hasSite, instrument_siteb3_id) self.RR.create_association(platform_site_id, PRED.hasSite, instrument_site2_id) self.RR.create_association(platform_site_id, PRED.hasSite, platform_siteb_id) self.RR.create_association(platform_siteb_id, PRED.hasSite, instrument_site_id) self.RR.create_association(platform_siteb_id, PRED.hasDevice, platform_deviceb_id) self.RR.create_association(platform_site_id, PRED.hasModel, platform_model_id) self.RR.create_association(platform_site_id, PRED.hasDevice, platform_device_id) self.RR.create_association(platform_site_id, PRED.hasDeployment, deployment_id) #instrument_site(s) self.RR.create_association(instrument_site_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_site_id, PRED.hasDevice, instrument_device_id) self.RR.create_association(instrument_site_id, PRED.hasDeployment, deployment_id) self.RR.create_association(instrument_site2_id, PRED.hasDevice, instrument_device2_id) #platform_device self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id) #instrument_device self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id) self.RR.create_association(instrument_device2_id, PRED.hasModel, instrument_model_id) ret = DotDict() ret.org_id = org_id ret.observatory_id = observatory_id ret.subsite_id = subsite_id ret.subsite2_id = subsite2_id ret.subsiteb_id = subsiteb_id ret.subsitez_id = subsitez_id ret.platform_site_id = platform_site_id ret.platform_siteb_id = platform_siteb_id ret.platform_siteb2_id = platform_siteb2_id ret.platform_site3_id = platform_site3_id ret.instrument_site_id = instrument_site_id ret.instrument_site2_id = instrument_site2_id ret.instrument_siteb3_id = instrument_siteb3_id ret.instrument_site4_id = instrument_site4_id ret.instrument_device_id = instrument_device_id ret.instrument_device2_id = instrument_device2_id ret.platform_device_id = platform_device_id ret.platform_deviceb_id = platform_deviceb_id ret.instrument_model_id = instrument_model_id ret.platform_model_id = platform_model_id ret.deployment_id = deployment_id return ret #@unittest.skip("targeting") def test_create_observatory(self): observatory_obj = IonObject(RT.Observatory, name='TestFacility', description='some new mf') observatory_id = self.OMS.create_observatory(observatory_obj) self.OMS.force_delete_observatory(observatory_id) #@unittest.skip("targeting") def test_find_observatory_org(self): org_obj = IonObject(RT.Org, name='TestOrg', description='some new mf org') org_id = self.OMS.create_marine_facility(org_obj) observatory_obj = IonObject(RT.Observatory, name='TestObservatory', description='some new obs') observatory_id = self.OMS.create_observatory(observatory_obj) #make association self.OMS.assign_resource_to_observatory_org(observatory_id, org_id) #find association org_objs = self.OMS.find_org_by_observatory(observatory_id) self.assertEqual(1, len(org_objs)) self.assertEqual(org_id, org_objs[0]._id) print("org_id=<" + org_id + ">") #create a subsite with parent Observatory subsite_obj = IonObject(RT.Subsite, name= 'TestSubsite', description = 'sample subsite') subsite_id = self.OMS.create_subsite(subsite_obj, observatory_id) self.assertIsNotNone(subsite_id, "Subsite not created.") # verify that Subsite is linked to Observatory mf_subsite_assoc = self.RR.get_association(observatory_id, PRED.hasSite, subsite_id) self.assertIsNotNone(mf_subsite_assoc, "Subsite not connected to Observatory.") # add the Subsite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=subsite_id, org_id=org_id) # verify that Subsite is linked to Org org_subsite_assoc = self.RR.get_association(org_id, PRED.hasResource, subsite_id) self.assertIsNotNone(org_subsite_assoc, "Subsite not connected as resource to Org.") #create a logical platform with parent Subsite platform_site_obj = IonObject(RT.PlatformSite, name= 'TestPlatformSite', description = 'sample logical platform') platform_site_id = self.OMS.create_platform_site(platform_site_obj, subsite_id) self.assertIsNotNone(platform_site_id, "PlatformSite not created.") # verify that PlatformSite is linked to Site site_lp_assoc = self.RR.get_association(subsite_id, PRED.hasSite, platform_site_id) self.assertIsNotNone(site_lp_assoc, "PlatformSite not connected to Site.") # add the PlatformSite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=platform_site_id, org_id=org_id) # verify that PlatformSite is linked to Org org_lp_assoc = self.RR.get_association(org_id, PRED.hasResource, platform_site_id) self.assertIsNotNone(org_lp_assoc, "PlatformSite not connected as resource to Org.") #create a logical instrument with parent logical platform instrument_site_obj = IonObject(RT.InstrumentSite, name= 'TestInstrumentSite', description = 'sample logical instrument') instrument_site_id = self.OMS.create_instrument_site(instrument_site_obj, platform_site_id) self.assertIsNotNone(instrument_site_id, "InstrumentSite not created.") # verify that InstrumentSite is linked to PlatformSite li_lp_assoc = self.RR.get_association(platform_site_id, PRED.hasSite, instrument_site_id) self.assertIsNotNone(li_lp_assoc, "InstrumentSite not connected to PlatformSite.") # add the InstrumentSite as a resource of this Observatory self.OMS.assign_resource_to_observatory_org(resource_id=instrument_site_id, org_id=org_id) # verify that InstrumentSite is linked to Org org_li_assoc = self.RR.get_association(org_id, PRED.hasResource, instrument_site_id) self.assertIsNotNone(org_li_assoc, "InstrumentSite not connected as resource to Org.") # remove the InstrumentSite as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(instrument_site_id, org_id) # verify that InstrumentSite is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.InstrumentSite, id_only=True ) self.assertEqual(len(assocs), 0) # remove the InstrumentSite self.OMS.delete_instrument_site(instrument_site_id) assocs, _ = self.RR.find_objects(platform_site_id, PRED.hasSite, RT.InstrumentSite, id_only=True ) self.assertEqual(len(assocs), 1) #todo: remove the dangling association # remove the PlatformSite as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(platform_site_id, org_id) # verify that PlatformSite is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.PlatformSite, id_only=True ) self.assertEqual(len(assocs), 0) # remove the Site as a resource of this Observatory self.OMS.unassign_resource_from_observatory_org(subsite_id, org_id) # verify that Site is linked to Org assocs,_ = self.RR.find_objects(org_id, PRED.hasResource, RT.Subsite, id_only=True ) self.assertEqual(len(assocs), 0) self.RR.delete(org_id) self.OMS.force_delete_observatory(observatory_id) self.OMS.force_delete_subsite(subsite_id) self.OMS.force_delete_platform_site(platform_site_id) self.OMS.force_delete_instrument_site(instrument_site_id) #@unittest.skip("in development...") @attr('EXT') def test_observatory_org_extended(self): stuff = self._make_associations() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product(input_resource_id=stuff.instrument_device_id, data_product_id=data_product_id1) #-------------------------------------------------------------------------------- # Get the extended Site (platformSite) #-------------------------------------------------------------------------------- extended_site = self.OMS.get_site_extension(stuff.platform_site_id) log.debug("extended_site: %s ", str(extended_site)) self.assertEqual(1, len(extended_site.platform_devices)) self.assertEqual(1, len(extended_site.platform_models)) self.assertEqual(stuff.platform_device_id, extended_site.platform_devices[0]._id) self.assertEqual(stuff.platform_model_id, extended_site.platform_models[0]._id) #-------------------------------------------------------------------------------- # Get the extended Org #-------------------------------------------------------------------------------- #test the extended resource extended_org = self.org_management_service.get_marine_facility_extension(stuff.org_id) log.debug("test_observatory_org_extended: extended_org: %s ", str(extended_org)) #self.assertEqual(2, len(extended_org.instruments_deployed) ) #self.assertEqual(1, len(extended_org.platforms_not_deployed) ) self.assertEqual(2, extended_org.number_of_platforms) self.assertEqual(2, len(extended_org.platform_models) ) self.assertEqual(2, extended_org.number_of_instruments) self.assertEqual(2, len(extended_org.instrument_models) ) #test the extended resource of the ION org ion_org_id = self.org_management_service.find_org() extended_org = self.org_management_service.get_marine_facility_extension(ion_org_id._id) log.debug("test_observatory_org_extended: extended_ION_org: %s ", str(extended_org)) self.assertEqual(0, len(extended_org.members)) self.assertEqual(0, extended_org.number_of_platforms) #self.assertEqual(1, len(extended_org.sites)) #-------------------------------------------------------------------------------- # Get the extended Site #-------------------------------------------------------------------------------- #create device state events to use for op /non-op filtering in extended t = get_ion_ts() self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device_id, state=ResourceAgentState.STREAMING ) self.event_publisher.publish_event( ts_created= t, event_type = 'ResourceAgentStateEvent', origin = stuff.instrument_device2_id, state=ResourceAgentState.INACTIVE ) extended_site = self.OMS.get_site_extension(stuff.instrument_site2_id) log.debug("test_observatory_org_extended: extended_site: %s ", str(extended_site)) self.dpclient.delete_data_product(data_product_id1)
class TestDataProductProvenance(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.dpmsclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.omsclient = ObservatoryManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient() self.dataset_management = DatasetManagementServiceClient() # deactivate all data processes when tests are complete def killAllDataProcesses(): for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]: self.dataprocessclient.deactivate_data_process(proc_id) self.dataprocessclient.delete_data_process(proc_id) self.addCleanup(killAllDataProcesses) def test_get_data_product_provenance_report(self): #Create a test device device_obj = Device(name='Device1', description='test instrument site') device_id, _ = self.rrclient.create(device_obj) self.addCleanup(self.rrclient.delete, device_id) #Create a test DataProduct data_product1_obj = DataProduct(name='DataProduct1', description='test data product 1') data_product1_id, _ = self.rrclient.create(data_product1_obj) self.addCleanup(self.rrclient.delete, data_product1_id) #Create a test DataProcess data_process_obj = DataProcess(name='DataProcess', description='test data process') data_process_id, _ = self.rrclient.create(data_process_obj) self.addCleanup(self.rrclient.delete, data_process_id) #Create a second test DataProduct data_product2_obj = DataProduct(name='DataProduct2', description='test data product 2') data_product2_id, _ = self.rrclient.create(data_product2_obj) self.addCleanup(self.rrclient.delete, data_product2_id) #Create a test DataProducer data_producer_obj = DataProducer(name='DataProducer', description='test data producer') data_producer_id, rev = self.rrclient.create(data_producer_obj) #Link the DataProcess to the second DataProduct manually assoc_id, _ = self.rrclient.create_association(subject=data_process_id, predicate=PRED.hasInputProduct, object=data_product2_id) self.addCleanup(self.rrclient.delete_association, assoc_id) # Register the instrument and process. This links the device and the data process # with their own producers self.damsclient.register_instrument(device_id) self.addCleanup(self.damsclient.unregister_instrument, device_id) self.damsclient.register_process(data_process_id) self.addCleanup(self.damsclient.unregister_process, data_process_id) #Manually link the first DataProduct with the test DataProducer assoc_id, _ = self.rrclient.create_association(subject=data_product1_id, predicate=PRED.hasDataProducer, object=data_producer_id) #Get the DataProducer linked to the DataProcess (created in register_process above) #Associate that with with DataProduct1's DataProducer data_process_producer_ids, _ = self.rrclient.find_objects(subject=data_process_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True) assoc_id, _ = self.rrclient.create_association(subject=data_process_producer_ids[0], predicate=PRED.hasParent, object=data_producer_id) self.addCleanup(self.rrclient.delete_association, assoc_id) #Get the DataProducer linked to the Device (created in register_instrument #Associate that with the DataProcess's DataProducer device_producer_ids, _ = self.rrclient.find_objects(subject=device_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True) assoc_id, _ = self.rrclient.create_association(subject=data_producer_id, predicate=PRED.hasParent, object=device_producer_ids[0]) #Create the links between the Device, DataProducts, DataProcess, and all DataProducers self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=data_product1_id) self.addCleanup(self.damsclient.unassign_data_product, device_id, data_product1_id) self.damsclient.assign_data_product(input_resource_id=data_process_id, data_product_id=data_product2_id) self.addCleanup(self.damsclient.unassign_data_product, data_process_id, data_product2_id) #Traverse through the relationships to get the links between objects res = self.dpmsclient.get_data_product_provenance_report(data_product2_id) #Make sure there are four keys self.assertEqual(len(res.keys()), 4) parent_count = 0 config_count = 0 for v in res.itervalues(): if 'parent' in v: parent_count += 1 if 'config' in v: config_count += 1 #Make sure there are three parents and four configs self.assertEqual(parent_count, 3) self.assertEqual(config_count, 4) @unittest.skip('This test is obsolete with new framework') def test_get_provenance(self): #create a deployment with metadata and an initial site and device instrument_site_obj = IonObject(RT.InstrumentSite, name='InstrumentSite1', description='test instrument site') instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "") log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id)) # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id)) self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id) # Create InstrumentAgent parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict' ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD, stream_configurations = [parsed_config] ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) #------------------------------- # Create CTD Parsed data product #------------------------------- tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id) log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition') dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) #------------------------------- # create a data product for the site to pass the OMS check.... we need to remove this check #------------------------------- dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id) #------------------------------- # Deploy instrument device to instrument site #------------------------------- deployment_obj = IonObject(RT.Deployment, name='TestDeployment', description='some new deployment') deployment_id = self.omsclient.create_deployment(deployment_obj) self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id) self.imsclient.deploy_instrument_device(instDevice_id, deployment_id) log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) ) self.omsclient.activate_deployment(deployment_id) inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False) log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) ) #------------------------------- # Create the agent instance #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestDataProductProvenance: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) log.debug("TestDataProductProvenance: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) log.debug("TestDataProductProvenance: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' ) outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' ) outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' ) log.debug("TestDataProductProvenance: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) log.debug("TestDataProductProvenance: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) log.debug("TestDataProductProvenance: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' ) outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' ) log.debug("TestDataProductProvenance: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) log.debug("TestDataProductProvenance: create output data product L2 Density") # ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, # name='L2_Density', # description='transform output pressure', # temporal_domain = tdom, # spatial_domain = sdom) # # ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, # outgoing_stream_l2_density_id, # parameter_dictionary) contactInfo = ContactInformation() contactInfo.individual_names_given = "Bill" contactInfo.individual_name_family = "Smith" contactInfo.street_address = "111 First St" contactInfo.city = "San Diego" contactInfo.email = "*****@*****.**" contactInfo.phones = ["858-555-6666"] contactInfo.country = "USA" contactInfo.postal_code = "92123" ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', contacts = [contactInfo], iso_topic_category = "my_iso_topic_category_here", quality_control_level = "1", temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L0 all data_process start") try: input_data_products = [ctd_parsed_data_product] output_data_products = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id] ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L0_all_dprocdef_id, in_data_product_ids = input_data_products, out_data_product_ids = output_data_products ) #activate only this data process just for coverage self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) contents = "this is the lookup table contents, replace with a file..." att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII) deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att) log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment) log.debug("TestDataProductProvenance: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_conductivity_dprocdef_id, in_data_product_ids = [ctd_l0_conductivity_output_dp_id], out_data_product_ids = [ctd_l1_conductivity_output_dp_id]) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_pressure_dprocdef_id, in_data_product_ids = [ctd_l0_pressure_output_dp_id], out_data_product_ids = [ctd_l1_pressure_output_dp_id]) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L1_temperature_dprocdef_id, in_data_product_ids = [ctd_l0_temperature_output_dp_id], out_data_product_ids = [ctd_l1_temperature_output_dp_id]) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L2_salinity_dprocdef_id, in_data_product_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], out_data_product_ids = [ctd_l2_salinity_output_dp_id]) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("TestDataProductProvenance: create L2_Density data_process start") try: in_dp_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id] out_dp_ids = [ctd_l2_density_output_dp_id] l2_density_all_data_process_id = self.dataprocessclient.create_data_process( data_process_definition_id = ctd_L2_density_dprocdef_id, in_data_product_ids = in_dp_ids, out_data_product_ids = out_dp_ids) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. # self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess()) # print 'activate_instrument: got ia client %s', self._ia_client # log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id) self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id) self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id) self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the provenance info for the ctd density data product #------------------------------- provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id) log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict)) #validate that products are represented self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)]) self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)]) density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)]) self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id]) #------------------------------- # Retrieve the extended resource for this data product #------------------------------- extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id) self.assertEqual(1, len(extended_product.data_processes) ) self.assertEqual(3, len(extended_product.process_input_data_products) ) # log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list)) # log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes)) # log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products)) # log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value)) #------------------------------- # Retrieve the extended resource for this data process #------------------------------- extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) # log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def)) # log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes)) # log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products)) self.assertEqual(1, len(extended_process_def.data_processes) ) self.assertEqual(3, len(extended_process_def.output_stream_definitions) ) self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process #------------------------------- # Request the xml report #------------------------------- results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id) print results #------------------------------- # Cleanup #------------------------------- self.dpmsclient.delete_data_product(ctd_parsed_data_product) self.dpmsclient.delete_data_product(log_data_product_id) self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id) self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id) self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestActivateInstrumentIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubcli = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dpclient = DataProductManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) #setup listerner vars self._data_greenlets = [] self._no_samples = None self._samples_received = [] @unittest.skip("TBD") def test_activateInstrumentSample(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) print 'new InstrumentModel id = ', instModel_id # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) print 'new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) driver_config = { 'dvr_mod' : 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls' : 'SBE37Driver', 'workdir' : '/tmp/', } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def) print 'new Stream Definition id = ', instDevice_id print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct,name='the parsed data',description='ctd stream test') try: data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new dp_id = ', data_product_id1 self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) print 'Data product streams1 = ', stream_ids print 'Creating new RAW data product with a stream definition' raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def) dp_obj = IonObject(RT.DataProduct,name='the raw data',description='raw stream test') try: data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) print 'new dp_id = ', data_product_id2 self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. #self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess()) print 'activate_instrument: got ia client %s', self._ia_client log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) print retval log.debug("test_activateInstrumentSample: initialize %s", str(retval)) time.sleep(2) log.debug("test_activateInstrumentSample: Sending go_active command (L4-CI-SA-RQ-334)") cmd = AgentCommand(command='go_active') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentSample: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state)) cmd = AgentCommand(command='run') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentSample: calling acquire_sample ") cmd = AgentCommand(command='acquire_sample') reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentSample: return from acquire_sample %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentSample: calling acquire_sample 2") cmd = AgentCommand(command='acquire_sample') reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentSample: return from acquire_sample 2 %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentSample: calling acquire_sample 3") cmd = AgentCommand(command='acquire_sample') reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentSample: return from acquire_sample 3 %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(2) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) def test_activateInstrumentStream(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) print 'new InstrumentModel id = ', instModel_id # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) print 'new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentStream: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_activateInstrumentStream: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) driver_config = { 'dvr_mod' : 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls' : 'SBE37Driver', 'workdir' : '/tmp/', } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def) log.debug( 'test_activateInstrumentStream new Stream Definition id = %s', instDevice_id ) log.debug( 'test_activateInstrumentStream Creating new CDM data product with a stream definition' ) dp_obj = IonObject(RT.DataProduct,name='the parsed data',description='ctd stream test') try: data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) log.debug( 'test_activateInstrumentStream new dp_id = %s', str(data_product_id1) ) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'test_activateInstrumentStream Data product streams1 = %s', str(stream_ids) ) simdata_subscription_id = self.pubsubcli.create_subscription( query=StreamQuery([stream_ids[0]]), exchange_name='Sim_data_queue', name='SimDataSubscription', description='SimData SubscriptionDescription' ) def simdata_message_received(message, headers): input = str(message) log.debug("test_activateInstrumentStream: granule received: %s", input) subscriber_registrar = StreamSubscriberRegistrar(process=self.container, node=self.container.node) simdata_subscriber = subscriber_registrar.create_subscriber(exchange_name='Sim_data_queue', callback=simdata_message_received) # Start subscribers simdata_subscriber.start() # Activate subscriptions self.pubsubcli.activate_subscription(simdata_subscription_id) log.debug( 'test_activateInstrumentStream Creating new RAW data product with a stream definition' ) raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def) dp_obj = IonObject(RT.DataProduct,name='the raw data',description='raw stream test') try: data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" %ex) log.debug( 'test_activateInstrumentStream new dp_id = %s', str(data_product_id2) ) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug( 'test_activateInstrumentStream Data product streams2 = %s', str(stream_ids) ) self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) log.debug( 'test_activateInstrumentStream Instrument agent instance obj: = %s', str(inst_agent_instance_obj) ) # Start a resource agent client to talk with the instrument agent. #self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess()) log.debug( 'test_activateInstrumentStream: got ia client %s', str(self._ia_client )) log.debug("test_activateInstrumentStream: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: initialize %s", str(retval)) time.sleep(2) log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)") cmd = AgentCommand(command='go_active') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state)) cmd = AgentCommand(command='run') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: run %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentStream: calling go_streaming ") cmd = AgentCommand(command='go_streaming') reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentStream: return from go_streaming %s", str(reply)) time.sleep(15) log.debug("test_activateInstrumentStream: calling go_observatory") cmd = AgentCommand(command='go_observatory') reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentStream: return from go_observatory %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentStream: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return from reset %s", str(reply)) time.sleep(2) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
class TestCTDTransformsIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node) def create_logger(self, name, stream_id=''): # logger process producer_definition = ProcessDefinition(name=name+'_logger') producer_definition.executable = { 'module':'ion.processes.data.stream_granule_logger', 'class':'StreamGranuleLogger' } logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition) configuration = { 'process':{ 'stream_id':stream_id, } } pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration) return pid def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_module='mi.instrument.seabird.sbe37smb.ooicore.driver', driver_class='SBE37Driver', comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) log.debug( 'new Stream Definition id = %s', instDevice_id) log.debug( 'Creating new CDM data product with a stream definition') craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids #------------------------------- # Create CTD Raw as the second data product #------------------------------- log.debug( 'Creating new RAW data product with a stream definition') raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary) log.debug( 'new dp_id = %s', str(ctd_raw_data_product)) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.debug( 'Data product streams2 = %s', str(stream_ids)) #todo: attaching the taxonomy to the stream is a TEMPORARY measure # Create taxonomies for both parsed and attach to the stream RawTax = TaxyTool() RawTax.add_taxonomy_set('raw_fixed','Fixed length bytes in an array of records') RawTax.add_taxonomy_set('raw_blob','Unlimited length bytes in an array') # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform', process_source='CTDL1ConductivityTransform source code here...') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform', process_source='CTDL1PressureTransform source code here...') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform', process_source='CTDL1TemperatureTransform source code here...') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform', process_source='DensityTransform source code here...') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products={} log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id, parameter_dictionary) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity = L1_conductivity_stream_definition() outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id ) outgoing_stream_l1_pressure = L1_pressure_stream_definition() outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id ) outgoing_stream_l1_temperature = L1_temperature_stream_definition() outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id ) log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] ) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id, parameter_dictionary ) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0] ) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0] ) self.loggerpids.append(pid) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id ) outgoing_stream_l2_density = L2_density_stream_definition() outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id ) log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0] ) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density") ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0] ) self.loggerpids.append(pid) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return") #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return") #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return") #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return") #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return") #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) print 'activate_instrument: got ia client %s', self._ia_client log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Streaming #------------------------------- cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: initialize %s", str(retval)) log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)") cmd = AgentCommand(command='go_active') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply)) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state)) cmd = AgentCommand(command='run') reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from run state: %s", str(state)) # Make sure the sampling rate and transmission are sane. params = { SBE37Parameter.NAVG : 1, SBE37Parameter.INTERVAL : 5, SBE37Parameter.TXREALTIME : True } self._ia_client.set_param(params) log.debug("test_activateInstrumentStream: calling go_streaming ") cmd = AgentCommand(command='go_streaming') reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state)) time.sleep(7) log.debug("test_activateInstrumentStream: calling go_observatory") cmd = AgentCommand(command='go_observatory') reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) log.debug("test_activateInstrumentStream: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result)) time.sleep(2) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid)
class TestCTDTransformsIntegration(IonIntegrationTestCase): def setUp(self): # Start container #print 'instantiating container' self._start_container() #container = Container() #print 'starting container' #container.start() #print 'started container' self.container.start_rel_from_url('res/deploy/r2deploy.yml') print 'started services' # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient( node=self.container.node) self.pubsubclient = PubsubManagementServiceClient( node=self.container.node) self.ingestclient = IngestionManagementServiceClient( node=self.container.node) self.imsclient = InstrumentManagementServiceClient( node=self.container.node) self.dataproductclient = DataProductManagementServiceClient( node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient( node=self.container.node) self.datasetclient = DatasetManagementServiceClient( node=self.container.node) def test_createTransformsThenActivateInstrument(self): # Set up the preconditions # ingestion configuration parameters self.exchange_point_id = 'science_data' self.number_of_workers = 2 self.hdf_storage = HdfStorage(relative_path='ingest') self.couch_storage = CouchStorage(datastore_name='test_datastore') self.XP = 'science_data' self.exchange_name = 'ingestion_queue' #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent") try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") try: instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- # driver_config = { # 'svr_addr': "localhost", # 'cmd_port': 5556, # 'evt_port': 5557, # 'dvr_mod': "ion.agents.instrument.drivers.sbe37.sbe37_driver", # 'dvr_cls': "SBE37Driver", # 'comms_config': { # 'addr': 'sbe37-simulator.oceanobservatories.org', # 'port': 4001, # } # } driver_config = { 'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls': 'SBE37Driver', 'workdir': '/tmp/', } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config=driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<', '>>']) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition( container=ctd_stream_def) print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', instDevice_id print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test') try: ctd_parsed_data_product = self.dataproductclient.create_data_product( dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids #------------------------------- # Create CTD Raw as the second data product #------------------------------- print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition' raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubclient.create_stream_definition( container=raw_stream_def) dp_obj = IonObject(RT.DataProduct, name='ctd_raw', description='raw stream test') try: ctd_raw_data_product = self.dataproductclient.create_data_product( dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) print 'new ctd_raw_data_product_id = ', ctd_raw_data_product self.damsclient.assign_data_product( input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug( "TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new ctd_L0_all data process definition: %s" % ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug( "TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform', process_source='CTDL1ConductivityTransform source code here...') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new CTDL1ConductivityTransform data process definition: %s" % ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug( "TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform', process_source='CTDL1PressureTransform source code here...') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new CTDL1PressureTransform data process definition: %s" % ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug( "TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform', process_source='CTDL1TemperatureTransform source code here...') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new CTDL1TemperatureTransform data process definition: %s" % ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug( "TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new SalinityTransform data process definition: %s" % ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug( "TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform" ) dpd_obj = IonObject( RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform', process_source='DensityTransform source code here...') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) except BadRequest as ex: self.fail( "failed to create new DensityTransform data process definition: %s" % ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id) self.output_products = {} log.debug( "test_createTransformsThenActivateInstrument: create output data product L0 conductivity" ) ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity') ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True) log.debug( "test_createTransformsThenActivateInstrument: create output data product L0 pressure" ) ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure') ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True) log.debug( "test_createTransformsThenActivateInstrument: create output data product L0 temperature" ) ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature') ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True) #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity = L1_conductivity_stream_definition() outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l1_conductivity, name='L1_conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id) outgoing_stream_l1_pressure = L1_pressure_stream_definition() outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l1_pressure, name='L1_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id) outgoing_stream_l1_temperature = L1_temperature_stream_definition() outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l1_temperature, name='L1_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id) log.debug( "test_createTransformsThenActivateInstrument: create output data product L1 conductivity" ) ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity') ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_l1_conductivity_output_dp_id, persist_data=True, persist_metadata=True) log.debug( "test_createTransformsThenActivateInstrument: create output data product L1 pressure" ) ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure') ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_l1_pressure_output_dp_id, persist_data=True, persist_metadata=True) log.debug( "test_createTransformsThenActivateInstrument: create output data product L1 temperature" ) ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature') ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_l1_temperature_output_dp_id, persist_data=True, persist_metadata=True) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l2_salinity, name='L2_salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id) outgoing_stream_l2_density = L2_density_stream_definition() outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition( container=outgoing_stream_l2_density, name='L2_Density') self.dataprocessclient.assign_stream_definition_to_data_process_definition( outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id) log.debug( "test_createTransformsThenActivateInstrument: create output data product L2 Salinity" ) ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity') ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_l2_salinity_output_dp_id, persist_data=True, persist_metadata=True) log.debug( "test_createTransformsThenActivateInstrument: create output data product L2 Density" ) ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure') ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product( ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id) self.dataproductclient.activate_data_product_persistence( data_product_id=ctd_l2_density_output_dp_id, persist_data=True, persist_metadata=True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug( "test_createTransformsThenActivateInstrument: create L0 all data_process start" ) try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) self.dataprocessclient.activate_data_process( ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "test_createTransformsThenActivateInstrument: create L0 all data_process return" ) #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug( "test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start" ) try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process( ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output': ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process( l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return" ) #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug( "test_createTransformsThenActivateInstrument: create L1_Pressure data_process start" ) try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process( ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output': ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process( l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "test_createTransformsThenActivateInstrument: create L1_Pressure data_process return" ) #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug( "test_createTransformsThenActivateInstrument: create L1_Pressure data_process start" ) try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output': ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process( l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "test_createTransformsThenActivateInstrument: create L1_Pressure data_process return" ) #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug( "test_createTransformsThenActivateInstrument: create L2_salinity data_process start" ) try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output': ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process( l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "test_createTransformsThenActivateInstrument: create L2_salinity data_process return" ) #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug( "test_createTransformsThenActivateInstrument: create L2_Density data_process start" ) try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process( ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output': ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process( l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" % ex) log.debug( "test_createTransformsThenActivateInstrument: create L2_Density data_process return" ) #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient( 'iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) print 'activate_instrument: got ia client %s', self._ia_client log.debug( " test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Streaming #------------------------------- cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) print retval log.debug( "test_createTransformsThenActivateInstrument:: initialize %s", str(retval)) time.sleep(1) cmd = AgentCommand(command='go_active') reply = self._ia_client.execute_agent(cmd) log.debug("test_createTransformsThenActivateInstrument:: go_active %s", str(reply)) time.sleep(1) cmd = AgentCommand(command='run') reply = self._ia_client.execute_agent(cmd) log.debug("test_createTransformsThenActivateInstrument:: run %s", str(reply)) time.sleep(1) # Make sure the sampling rate and transmission are sane. params = { SBE37Parameter.NAVG: 1, SBE37Parameter.INTERVAL: 5, SBE37Parameter.TXREALTIME: True } self._ia_client.set_param(params) self._no_samples = 2 # Begin streaming. cmd = AgentCommand(command='go_streaming') reply = self._ia_client.execute_agent(cmd) log.debug( "test_createTransformsThenActivateInstrument:: go_streaming %s", str(reply)) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) time.sleep(15) log.debug("test_activateInstrument: calling go_observatory") cmd = AgentCommand(command='go_observatory') reply = self._ia_client.execute(cmd) log.debug("test_activateInstrument: return from go_observatory %s", str(reply)) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) time.sleep(2) log.debug( "test_createTransformsThenActivateInstrument:: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client.execute_agent(cmd) log.debug( "test_createTransformsThenActivateInstrument:: return from reset %s", str(reply)) time.sleep(2) #------------------------------- # Sampling #------------------------------- # cmd = AgentCommand(command='initialize') # retval = self._ia_client.execute_agent(cmd) # print retval # log.debug("test_createTransformsThenActivateInstrument:: initialize %s", str(retval)) # time.sleep(2) # # cmd = AgentCommand(command='go_active') # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activateInstrument: go_active %s", str(reply)) # time.sleep(2) # # cmd = AgentCommand(command='run') # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activateInstrument: run %s", str(reply)) # time.sleep(2) # # log.debug("test_activateInstrument: calling acquire_sample ") # cmd = AgentCommand(command='acquire_sample') # reply = self._ia_client.execute(cmd) # log.debug("test_activateInstrument: return from acquire_sample %s", str(reply)) # time.sleep(2) # # log.debug("test_activateInstrument: calling acquire_sample 2") # cmd = AgentCommand(command='acquire_sample') # reply = self._ia_client.execute(cmd) # log.debug("test_activateInstrument: return from acquire_sample 2 %s", str(reply)) # time.sleep(2) # # log.debug("test_activateInstrument: calling acquire_sample 3") # cmd = AgentCommand(command='acquire_sample') # reply = self._ia_client.execute(cmd) # log.debug("test_activateInstrument: return from acquire_sample 3 %s", str(reply)) # time.sleep(2) # # log.debug("test_activateInstrument: calling go_inactive ") # cmd = AgentCommand(command='go_inactive') # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activateInstrument: return from go_inactive %s", str(reply)) # time.sleep(2) # # log.debug("test_activateInstrument: calling reset ") # cmd = AgentCommand(command='reset') # reply = self._ia_client.execute_agent(cmd) # log.debug("test_activateInstrument: return from reset %s", str(reply)) # time.sleep(2) # # # self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) # # # #get the dataset id of the ctd_parsed product from the dataproduct ctd_parsed_data_product # ctd_parsed_data_product_obj = self.dataproductclient.read_data_product(ctd_parsed_data_product) # log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset id %s", str(ctd_parsed_data_product_obj.dataset_id)) # # # ask for the dataset bounds from the datasetmgmtsvc # bounds = self.datasetclient.get_dataset_bounds(ctd_parsed_data_product_obj.dataset_id) # log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset bounds %s", str(bounds)) # print 'activate_instrument: got dataset bounds %s', str(bounds) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id)