def await_state(self, state, timeout=10): print "Emptying event queue" while True: event = self.latest_event(0) if event: print "State %s from event %s" % (event.state, event) else: break self.event_sub.start() #wait for process state print "Setting up %s gate" % ProcessStateEnum._str_map[state] gate = ProcessStateGate(self.pd_cli.read_process, self.pid, state) print "Waiting" ret = gate.await(timeout) print "Await got %s" % ret event = self.latest_event(timeout=1) # check false positives/negatives if ret and gate._get_first_chance() is None and event is None: self.fail("ProcessStateGate got an event that EventSubscriber didnt....") self.event_sub.stop() if (not ret) or gate._get_last_chance(): if event and event.state == state: self.fail("EventSubscriber got state event %s for process %s, ProcessStateGate missed it" % (ProcessStateEnum._str_map[event.state], self.pid)) return ret
def await_state(self, state, timeout=10): print "Emptying event queue" while True: event = self.latest_event(0) if event: print "State %s from event %s" % (event.state, event) else: break self.event_sub.start() #wait for process state print "Setting up %s gate" % ProcessStateEnum._str_map[state] gate = ProcessStateGate(self.pd_cli.read_process, self.pid, state) print "Waiting" ret = gate. await (timeout) print "Await got %s" % ret event = self.latest_event(timeout=1) # check false positives/negatives if ret and gate._get_first_chance() is None and event is None: self.fail( "ProcessStateGate got an event that EventSubscriber didnt....") self.event_sub.stop() if (not ret) or gate._get_last_chance(): if event and event.state == state: self.fail( "EventSubscriber got state event %s for process %s, ProcessStateGate missed it" % (ProcessStateEnum._str_map[event.state], self.pid)) return ret
def sync_launch(self, config): self.process_dispatcher.schedule_process(self.process_definition_id, process_id=self.process_id, configuration=config) gate = ProcessStateGate(self.process_dispatcher.read_process, self.process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate. await (30)) self.addCleanup(self.process_dispatcher.cancel_process, self.process_id)
def await_launch(self, timeout, process_id=None): if None is process_id: if None is self.process_id: raise BadRequest("No process_id was supplied to await_launch, and " + "no process_id was available from launch") else: process_id = self.process_id log.debug("waiting %s seconds for agent launch", timeout) psg = ProcessStateGate(self.process_dispatcher_client.read_process, process_id, ProcessStateEnum.RUNNING) if not psg.await(20): # todo: different error raise BadRequest("The agent process '%s' failed to launch in %s seconds" % (process_id, timeout))
def await_agent_ready(self, replay_timeout=5): ''' Determines if the process has been started @param replay_timeout Time to wait before raising a timeout @retval True if the process has been started ''' if self.process: pd_cli = ProcessDispatcherServiceProcessClient( process=self.process) else: pd_cli = ProcessDispatcherServiceClient() process_gate = ProcessStateGate(pd_cli.read_process, self.replay_process_id, ProcessStateEnum.RUNNING) return process_gate. await (replay_timeout)
def _do_launch_gate(self, platform_id, agent_config, timeout_spawn): """ The method for when using the ProcessStateGate pattern, which is the one used by test_oms_launch2 to launch the root platform. """ log.debug("_do_launch_gate: platform_id=%r, timeout_spawn=%s", platform_id, timeout_spawn) pa_name = 'PlatformAgent_%s' % platform_id pdef = ProcessDefinition(name=pa_name) pdef.executable = {'module': PA_MOD, 'class': PA_CLS} pdef_id = self._pd_client.create_process_definition( process_definition=pdef) log.debug("using schedule_process directly %r", platform_id) pid = self._pd_client.schedule_process(process_definition_id=pdef_id, schedule=None, configuration=agent_config) if timeout_spawn: # ProcessStateGate used as indicated in its pydoc (9/21/12) gate = ProcessStateGate(self._pd_client.read_process, pid, ProcessStateEnum.RUNNING) err_msg = None try: if not gate. await (timeout_spawn): err_msg = "The platform agent instance did not spawn in " \ "%s seconds. gate.wait returned false. " % \ timeout_spawn log.error(err_msg) except Exception as e: log.error( "Exception while waiting for platform agent instance " "(platform_id=%r) " "to spawn in %s seconds: %s", platform_id, timeout_spawn, str(e)) #,exc_Info=True) if err_msg: raise PlatformException(err_msg) log.debug( "_do_launch_gate: platform_id=%r: agent spawned, pid=%r " "(ProcessStateGate pattern used)", platform_id, pid) return pid
def _launch_highcharts(self, viz_id, data_product_id, out_stream_id): ''' Launches the high-charts transform ''' stream_ids, _ = self.clients.resource_registry.find_objects( data_product_id, PRED.hasStream, id_only=True) if not stream_ids: raise BadRequest( "Can't launch high charts streaming: data product doesn't have associated stream (%s)" % data_product_id) queue_name = 'viz_%s' % data_product_id sub_id = self.clients.pubsub_management.create_subscription( name='viz transform for %s' % data_product_id, exchange_name=queue_name, stream_ids=stream_ids) self.clients.pubsub_management.activate_subscription(sub_id) self.clients.resource_registry.create_association( viz_id, PRED.hasSubscription, sub_id) config = DotDict() config.process.publish_streams.highcharts = out_stream_id config.process.queue_name = queue_name # This process MUST be launched the first time or fail so the user # doesn't wait there for nothing to happen. schedule = ProcessSchedule() schedule.restart_mode = ProcessRestartMode.NEVER schedule.queueing_mode = ProcessQueueingMode.NEVER # Launch the process procdef_id = self._get_highcharts_procdef() pid = self.clients.process_dispatcher.schedule_process( process_definition_id=procdef_id, schedule=schedule, configuration=config) # Make sure it launched or raise an error process_gate = ProcessStateGate( self.clients.process_dispatcher.read_process, pid, ProcessStateEnum.RUNNING) if not process_gate. await (self.CFG.get_safe( 'endpoint.receive.timeout', 10)): raise ServiceUnavailable( "Failed to launch high charts realtime visualization")
def cancel_process(self, pid, timeout_cancel=None): """ Helper to terminate a process """ pinfo = self._pd_client.read_process(pid) if pinfo.process_state != ProcessStateEnum.RUNNING: log.debug("cancel_process: pid=%r is not RUNNING", pid) return log.debug("cancel_process: canceling pid=%r", pid) self._pd_client.cancel_process(pid) if timeout_cancel: log.debug("waiting %s seconds for preocess to cancel", timeout_cancel) psg = ProcessStateGate(self._pd_client.read_process, pid, ProcessStateEnum.TERMINATED) if not psg.await(timeout_cancel): log.debug("Process %r failed to get to TERMINATED in %s seconds", pid, timeout_cancel)
def test_activateInstrumentSample(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) log.debug('new InstrumentModel id = %s ', instModel_id) #Create stream alarms """ test_two_sided_interval Test interval alarm and alarm event publishing for a closed inteval. """ # kwargs = { # 'name' : 'test_sim_warning', # 'stream_name' : 'parsed', # 'value_id' : 'temp', # 'message' : 'Temperature is above test range of 5.0.', # 'type' : StreamAlarmType.WARNING, # 'upper_bound' : 5.0, # 'upper_rel_op' : '<' # } kwargs = { 'name': 'temperature_warning_interval', 'stream_name': 'parsed', 'value_id': 'temp', 'message': 'Temperature is below the normal range of 50.0 and above.', 'type': StreamAlarmType.WARNING, 'lower_bound': 50.0, 'lower_rel_op': '<' } # Create alarm object. alarm = {} alarm['type'] = 'IntervalAlarmDef' alarm['kwargs'] = kwargs raw_config = StreamConfiguration( stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5) parsed_config = StreamConfiguration( stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5, alarms=[alarm]) # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri= "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg", stream_configurations=[raw_config, parsed_config]) instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) log.debug('new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) log.debug( "test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug('new dp_id = %s', data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0]) self.loggerpids.append(pid) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug('new dp_id = %s', data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # setup notifications for the device and parsed data product user_id_1 = self._create_notification(user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1) #---------- Create notifications for another user and verify that we see different computed subscriptions for the two users --------- user_id_2 = self._create_notification(user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('Data product streams2 = %s', str(stream_ids)) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id2 = %s', dataset_ids[0]) self.raw_dataset = dataset_ids[0] #elastic search debug es_indexes, _ = self.container.resource_registry.find_resources( restype='ElasticSearchIndex') log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes]) log.debug('Bootstrap %s', CFG.bootstrap.use_es) def start_instrument_agent(): self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) gevent.joinall([gevent.spawn(start_instrument_agent)]) #setup a subscriber to alarm events from the device self._events_received = [] self._event_count = 0 self._samples_out_of_range = 0 self._samples_complete = False self._async_sample_result = AsyncResult() def consume_event(*args, **kwargs): log.debug( 'TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) self._event_count = len(self._events_received) self._async_sample_result.set() self._event_subscriber = EventSubscriber( event_type= 'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent callback=consume_event, origin=instDevice_id) self._event_subscriber.start() #cleanup self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) def stop_subscriber(): self._event_subscriber.stop() self._event_subscriber = None self.addCleanup(stop_subscriber) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % inst_agent_instance_obj.agent_process_id) log.debug('Instrument agent instance obj: = %s', str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient( instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: initialize %s", str(retval)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: return value from go_active %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: run %s", str(reply)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) retval = self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE) for i in xrange(10): retval = self._ia_client.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from sample %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) self._samples_complete = True #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt.pretty_print())) temp_vals = rdt['temp'] self.assertEquals(len(temp_vals), 10) log.debug("test_activateInstrumentSample: all temp_vals: %s", temp_vals) #out_of_range_temp_vals = [i for i in temp_vals if i > 5] out_of_range_temp_vals = [i for i in temp_vals if i < 50.0] log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s", out_of_range_temp_vals) self._samples_out_of_range = len(out_of_range_temp_vals) # if no bad values were produced, then do not wait for an event if self._samples_out_of_range == 0: self._async_sample_result.set() log.debug("test_activateInstrumentSample: _events_received: %s", self._events_received) log.debug("test_activateInstrumentSample: _event_count: %s", self._event_count) self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout) replay_data = self.dataretrieverclient.retrieve(self.raw_dataset) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) log.debug("RDT raw: %s", str(rdt.pretty_print())) raw_vals = rdt['raw'] self.assertEquals(len(raw_vals), 10) log.debug("l4-ci-sa-rq-138") """ Physical resource control shall be subject to policy Instrument management control capabilities shall be subject to policy The actor accessing the control capabilities must be authorized to send commands. note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to get an example of a policy test and use that to create a test stub that will be completed when we have instrument policies. Tim M: The "actor", aka observatory operator, will access the instrument through ION. """ #-------------------------------------------------------------------------------- # Get the extended data product to see if it contains the granules #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id1, user_id=user_id_1) def poller(extended_product): return len(extended_product.computed.user_notification_requests. value) == 1 poll(poller, extended_product, timeout=30) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id1, extended_data_product=extended_product) #-------------------------------------------------------------------------------- #put some events into the eventsdb to test - this should set the comms and data status to WARNING #-------------------------------------------------------------------------------- t = get_ion_ts() self.event_publisher.publish_event(ts_created=t, event_type='DeviceStatusEvent', origin=instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values=[200]) self.event_publisher.publish_event( ts_created=t, event_type='DeviceCommsEvent', origin=instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds=20) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_1) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # For the second user, check the extended data product and the extended intrument #-------------------------------------------------------------------------------- extended_product = self.dpclient.get_data_product_extension( data_product_id=data_product_id2, user_id=user_id_2) self._check_computed_attributes_of_extended_product( expected_data_product_id=data_product_id2, extended_data_product=extended_product) #---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING --------- t = get_ion_ts() self.event_publisher.publish_event(ts_created=t, event_type='DeviceStatusEvent', origin=instDevice_id, state=DeviceStatusType.OUT_OF_RANGE, values=[200]) self.event_publisher.publish_event( ts_created=t, event_type='DeviceCommsEvent', origin=instDevice_id, state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION, lapse_interval_seconds=20) #-------------------------------------------------------------------------------- # Get the extended instrument #-------------------------------------------------------------------------------- extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id, user_id=user_id_2) self._check_computed_attributes_of_extended_instrument( expected_instrument_device_id=instDevice_id, extended_instrument=extended_instrument) #-------------------------------------------------------------------------------- # Deactivate loggers #-------------------------------------------------------------------------------- for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid) self.dpclient.delete_data_product(data_product_id1) self.dpclient.delete_data_product(data_product_id2)
def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True): """ This method runs a test of launching a driver with a given agent configuration """ # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.imsclient.create_instrument_model(instModel_obj) print 'new InstrumentModel id = %s ' % instModel_id # Create InstrumentAgent instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) print 'new InstrumentAgent id = %s' % instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config=port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_raw_param_dict', id_only=True) parsed_stream_def_id = self.pubsubcli.create_stream_definition( name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_stream_def_id = self.pubsubcli.create_stream_definition( name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new dp_id = %s' % data_product_id1 self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) print 'Data product streams1 = %s' % stream_ids # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id1 = %s' % dataset_ids[0] self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data self.get_datastore(self.parsed_dataset) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain=tdom, spatial_domain=sdom) data_product_id2 = self.dpclient.create_data_product( data_product=dp_obj, stream_definition_id=raw_stream_def_id) print 'new dp_id = %s' % str(data_product_id2) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) print 'Data product streams2 = %s' % str(stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True) print 'Data set for data_product_id2 = %s' % dataset_ids[0] self.raw_dataset = dataset_ids[0] # add start/stop for instrument agent gevent.joinall([ gevent.spawn( lambda: self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id)) ]) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) agent_process_id = ResourceAgentClient._get_agent_process_id( instDevice_id) print "Agent process id is '%s'" % str(agent_process_id) self.assertTrue(agent_process_id) gate = ProcessStateGate(self.processdispatchclient.read_process, agent_process_id, ProcessStateEnum.RUNNING) if not expect_launch: self.assertFalse( gate. await (30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" % agent_process_id) return self.assertTrue( gate. await (30), "The instrument agent instance (%s) did not spawn in 30 seconds" % agent_process_id) print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING" #print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(instDevice_id, to_name=agent_process_id, process=FakeProcess()) print "ResourceAgentClient created: %s" % str(self._ia_client) print "Sending command=ResourceAgentEvent.INITIALIZE" cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) if not expect_command: self.assertRaises(ServerError, self._ia_client.execute_agent, cmd) return retval = self._ia_client.execute_agent(cmd) print "Result of INITIALIZE: %s" % str(retval) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertTrue(state, 'DRIVER_STATE_COMMAND') cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client.execute_agent(cmd) self.assertTrue(reply.status == 0) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) # This gevent sleep is there to test the autosample time, which will show something different from default # only if the instrument runs for over a minute gevent.sleep(90) extended_instrument = self.imsclient.get_instrument_device_extension( instrument_device_id=instDevice_id) self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue) autosample_string = extended_instrument.computed.uptime.value autosampling_time = int(autosample_string.split()[4]) self.assertTrue(autosampling_time > 0) cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client.execute_resource(cmd) print "Sending command=ResourceAgentEvent.RESET" cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client.execute_agent(cmd) print "Result of RESET: %s" % str(reply)
def test_single_platform(self): full_config = self._create_platform_configuration() platform_id = 'LJ01D' stream_configurations = self.get_streamConfigs() agent__obj = IonObject(RT.PlatformAgent, name='%s_PlatformAgent' % platform_id, description='%s_PlatformAgent platform agent' % platform_id, stream_configurations=stream_configurations) agent_id = self.IMS.create_platform_agent(agent__obj) device__obj = IonObject( RT.PlatformDevice, name='%s_PlatformDevice' % platform_id, description='%s_PlatformDevice platform device' % platform_id, # ports=port_objs, # platform_monitor_attributes = monitor_attribute_objs ) self.device_id = self.IMS.create_platform_device(device__obj) ####################################### # data product (adapted from test_instrument_management_service_integration) tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='DataProduct test', processing_level_code='Parsed_Canonical', temporal_domain=tdom, spatial_domain=sdom) data_product_id1 = self.DP.create_data_product( data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id) log.debug('data_product_id1 = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=self.device_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence( data_product_id=data_product_id1) ####################################### ####################################### # dataset stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug('Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] ####################################### full_config['platform_config'] = { 'platform_id': platform_id, 'driver_config': DVR_CONFIG, 'network_definition': self._network_definition_ser } agent_instance_obj = IonObject( RT.PlatformAgentInstance, name='%s_PlatformAgentInstance' % platform_id, description="%s_PlatformAgentInstance" % platform_id, agent_config=full_config) agent_instance_id = self.IMS.create_platform_agent_instance( platform_agent_instance=agent_instance_obj, platform_agent_id=agent_id, platform_device_id=self.device_id) stream_id = stream_ids[0] self._start_data_subscriber(agent_instance_id, stream_id) log.debug( "about to call imsclient.start_platform_agent_instance with id=%s", agent_instance_id) pid = self.IMS.start_platform_agent_instance( platform_agent_instance_id=agent_instance_id) log.debug("start_platform_agent_instance returned pid=%s", pid) #wait for start instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (90), "The platform agent instance did not spawn in 90 seconds") agent_instance_obj = self.IMS.read_instrument_agent_instance( agent_instance_id) log.debug('Platform agent instance obj') # Start a resource agent client to talk with the instrument agent. self._pa_client = ResourceAgentClient( 'paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug("got platform agent client %s", str(self._pa_client)) # ping_agent can be issued before INITIALIZE retval = self._pa_client.ping_agent(timeout=TIMEOUT) log.debug('Base Platform ping_agent = %s', str(retval)) cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform INITIALIZE = %s', str(retval)) # GO_ACTIVE cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform GO_ACTIVE = %s', str(retval)) # RUN: cmd = AgentCommand(command=PlatformAgentEvent.RUN) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform RUN = %s', str(retval)) # START_MONITORING: cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform START_MONITORING = %s', str(retval)) # wait for data sample # just wait for at least one -- see consume_data above log.info("waiting for reception of a data sample...") self._async_data_result.get(timeout=DATA_TIMEOUT) self.assertTrue(len(self._samples_received) >= 1) log.info("waiting a bit more for reception of more data samples...") sleep(15) log.info("Got data samples: %d", len(self._samples_received)) # wait for event # just wait for at least one event -- see consume_event above log.info("waiting for reception of an event...") self._async_event_result.get(timeout=EVENT_TIMEOUT) log.info("Received events: %s", len(self._events_received)) #get the extended platfrom which wil include platform aggreate status fields # extended_platform = self.IMS.get_platform_device_extension(self.device_id) # log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) ) # log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) ) # log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) ) # STOP_MONITORING: cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform STOP_MONITORING = %s', str(retval)) # GO_INACTIVE cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform GO_INACTIVE = %s', str(retval)) # RESET: Resets the base platform agent, which includes termination of # its sub-platforms processes: cmd = AgentCommand(command=PlatformAgentEvent.RESET) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform RESET = %s', str(retval)) #------------------------------- # Stop Base Platform AgentInstance #------------------------------- self.IMS.stop_platform_agent_instance( platform_agent_instance_id=agent_instance_id)
def test_createDataProcessUsingSim(self): #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" ) instModel_id = self.imsclient.create_instrument_model(instModel_obj) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg") instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) #------------------------------- # Create InstrumentAgentInstance to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4001, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'command_port': 4002, 'data_port': 4003, 'log_level': 5, } port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) #------------------------------- # Create CTD Parsed as the first data product #------------------------------- # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id) # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject(RT.DataProduct, name='ctd_parsed', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) #------------------------------- # Create CTD Raw as the second data product #------------------------------- raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id) dp_obj.name = 'ctd_raw' ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #------------------------------- # Create listener for data process events and verify that events are received. #------------------------------- # todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events #todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) data_process = self.rrclient.read(ctd_l0_all_data_process_id) process_id = data_process.process_id self.addCleanup(self.process_dispatcher.cancel_process, process_id) #------------------------------- # Wait until the process launched in the create_data_process() method is actually running, before proceeding further in this test #------------------------------- gate = ProcessStateGate(self.process_dispatcher.read_process, process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The data process (%s) did not spawn in 30 seconds" % process_id) #------------------------------- # Retrieve a list of all data process defintions in RR and validate that the DPD is listed #------------------------------- # todo: Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions # todo: data topics are being handled by pub sub at the level of streams self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) #todo: check that activate event is received L4-CI-SA-RQ-367 #todo... (it looks like no event is being published when the data process is activated... so below, we just check for now # todo... that the subscription is indeed activated) (Swarbhanu) # todo: monitor process to see if it is active (sa-rq-182) ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id) input_subscription_id = ctd_l0_all_data_process.input_subscription_id subs = self.rrclient.read(input_subscription_id) self.assertTrue(subs.activated) # todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id) #------------------------------- # Retrieve the extended resources for data process definition and for data process #------------------------------- extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id) self.assertEqual(1, len(extended_process_definition.data_processes)) log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition)) extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id) self.assertEqual(1, len(extended_process.input_data_products)) log.debug("test_createDataProcess: extended_process %s", str(extended_process)) ################################ Test the removal of data processes ################################## #------------------------------------------------------------------- # Test the deleting of the data process #------------------------------------------------------------------- # Before deleting, get the input streams, output streams and the subscriptions so that they can be checked after deleting # dp_obj_1 = self.rrclient.read(ctd_l0_all_data_process_id) # input_subscription_id = dp_obj_1.input_subscription_id # out_prods, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) # in_prods, _ = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasInputProduct, id_only=True) # in_streams = [] # for in_prod in in_prods: # streams, _ = self.rrclient.find_objects(in_prod, PRED.hasStream, id_only=True) # in_streams.extend(streams) # out_streams = [] # for out_prod in out_prods: # streams, _ = self.rrclient.find_objects(out_prod, PRED.hasStream, id_only=True) # out_streams.extend(streams) # Deleting the data process self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id) # Check that the data process got removed. Check the lcs state. It should be retired dp_obj = self.rrclient.read(ctd_l0_all_data_process_id) self.assertEquals(dp_obj.lcstate, LCS.RETIRED) # Check for process defs still attached to the data process dpd_assn_ids = self.rrclient.find_associations(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcessDefinition, id_only=True) self.assertEquals(len(dpd_assn_ids), 0) # Check for output data product still attached to the data process out_products, assocs = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True) self.assertEquals(len(out_products), 0) self.assertEquals(len(assocs), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check for input data products still attached to the data process inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct) self.assertEquals(len(inprod_associations), 0) # Check of the data process has been deactivated self.assertIsNone(dp_obj.input_subscription_id) # Read the original subscription id of the data process and check that it has been deactivated with self.assertRaises(NotFound): self.pubsubclient.read_subscription(input_subscription_id) #------------------------------------------------------------------- # Delete the data process definition #------------------------------------------------------------------- # before deleting, get the process definition being associated to in order to be able to check later if the latter gets deleted as it should proc_def_ids, proc_def_asocs = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasProcessDefinition) self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id) # check that the data process definition has been retired dp_proc_def = self.rrclient.read(ctd_L0_all_dprocdef_id) self.assertEquals(dp_proc_def.lcstate, LCS.RETIRED) # Check for old associations of this data process definition proc_defs, proc_def_asocs = self.rrclient.find_objects(ctd_L0_all_dprocdef_id, PRED.hasProcessDefinition) self.assertEquals(len(proc_defs), 0) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject= ctd_L0_all_dprocdef_id, id_only=True) self.assertEquals(len(obj_assns), 0) ################################ Test the removal of data processes ################################## # Try force delete... This should simply delete the associations and the data process object # from the resource registry #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id) #--------------------------------------------------------------------------------------------------------------- # Force deleting a data process definition #--------------------------------------------------------------------------------------------------------------- self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id) # find all associations where this is the subject _, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True) # find all associations where this is the object _, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True) self.assertEquals(len(obj_assns), 0) self.assertEquals(len(sbj_assns), 0) with self.assertRaises(NotFound): self.rrclient.read(ctd_l0_all_data_process_id)
def _create_launch_verify(self, base_platform_id): # and trigger the traversal of the branch rooted at that base platform # to create corresponding ION objects and configuration dictionaries: pnode = self._network_definition.pnodes[base_platform_id] base_platform_objs = self._traverse(pnode, base_platform_id) # now that most of the topology information is there, add the # PlatformAgentInstance elements self._set_platform_agent_instances() base_platform_config = self.platform_configs[base_platform_id] log.info("base_platform_id = %r", base_platform_id) #------------------------------------------------------------------------------------- # Create Data Process Definition and Data Process for the eng stream monitor process #------------------------------------------------------------------------------------- dpd_obj = IonObject( RT.DataProcessDefinition, name='DemoStreamAlertTransform', description='For testing EventTriggeredTransform_B', module='ion.processes.data.transforms.event_alert_transform', class_name='DemoStreamAlertTransform') self.platform_dprocdef_id = self.dataprocessclient.create_data_process_definition( dpd_obj) #THERE SHOULD BE NO STREAMDEF REQUIRED HERE. platform_streamdef_id = self.pubsubcli.create_stream_definition( name='platform_eng_parsed', parameter_dictionary_id=self.pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition( platform_streamdef_id, self.platform_dprocdef_id, binding='output') config = { 'process': { 'timer_interval': 5, 'queue_name': 'a_queue', 'variable_name': 'input_voltage', 'time_field_name': 'preferred_timestamp', 'valid_values': [-100, 100], 'timer_origin': 'Interval Timer' } } platform_data_process_id = self.dataprocessclient.create_data_process( self.platform_dprocdef_id, [self.data_product_id], {}, config) self.dataprocessclient.activate_data_process(platform_data_process_id) self.addCleanup(self.dataprocessclient.delete_data_process, platform_data_process_id) #------------------------------- # Launch Base Platform AgentInstance, connect to the resource agent client #------------------------------- agent_instance_id = base_platform_objs['agent_instance_id'] log.debug( "about to call imsclient.start_platform_agent_instance with id=%s", agent_instance_id) pid = self.imsclient.start_platform_agent_instance( platform_agent_instance_id=agent_instance_id) log.debug("start_platform_agent_instance returned pid=%s", pid) #wait for start instance_obj = self.imsclient.read_platform_agent_instance( agent_instance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue( gate. await (90), "The platform agent instance did not spawn in 90 seconds") agent_instance_obj = self.imsclient.read_instrument_agent_instance( agent_instance_id) log.debug( 'test_oms_create_and_launch: Platform agent instance obj: %s', str(agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. self._pa_client = ResourceAgentClient( 'paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess()) log.debug(" test_oms_create_and_launch:: got pa client %s", str(self._pa_client)) log.debug("base_platform_config =\n%s", base_platform_config) # ping_agent can be issued before INITIALIZE retval = self._pa_client.ping_agent(timeout=TIMEOUT) log.debug('Base Platform ping_agent = %s', str(retval)) # issue INITIALIZE command to the base platform, which will launch the # creation of the whole platform hierarchy rooted at base_platform_config['platform_id'] # cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE, kwargs=dict(plat_config=base_platform_config)) cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform INITIALIZE = %s', str(retval)) # GO_ACTIVE cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform GO_ACTIVE = %s', str(retval)) # RUN: cmd = AgentCommand(command=PlatformAgentEvent.RUN) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform RUN = %s', str(retval)) # START_MONITORING: cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform START_MONITORING = %s', str(retval)) # wait for data sample # just wait for at least one -- see consume_data above log.info("waiting for reception of a data sample...") self._async_data_result.get(timeout=DATA_TIMEOUT) self.assertTrue(len(self._samples_received) >= 1) log.info("waiting a bit more for reception of more data samples...") sleep(15) log.info("Got data samples: %d", len(self._samples_received)) # wait for event # just wait for at least one event -- see consume_event above log.info("waiting for reception of an event...") self._async_event_result.get(timeout=EVENT_TIMEOUT) log.info("Received events: %s", len(self._events_received)) #get the extended platfrom which wil include platform aggreate status fields extended_platform = self.imsclient.get_platform_device_extension( self.device_id) # log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) ) # log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) ) # log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) ) # STOP_MONITORING: cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform STOP_MONITORING = %s', str(retval)) # GO_INACTIVE cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform GO_INACTIVE = %s', str(retval)) # RESET: Resets the base platform agent, which includes termination of # its sub-platforms processes: cmd = AgentCommand(command=PlatformAgentEvent.RESET) retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT) log.debug('Base Platform RESET = %s', str(retval)) #------------------------------- # Stop Base Platform AgentInstance #------------------------------- self.imsclient.stop_platform_agent_instance( platform_agent_instance_id=agent_instance_id)
def test_deploy_activate_full(self): # ensure no processes or pids are left around by agents or Sims #self.cleanupprocs() self.loggerpids = [] #------------------------------- # Create InstrumentModel #------------------------------- instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) #------------------------------- # Create InstrumentAgent #------------------------------- instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg") try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) #------------------------------- # Create Instrument Site #------------------------------- instrumentSite_obj = IonObject(RT.InstrumentSite, name='instrumentSite1', description="SBE37IMInstrumentSite" ) try: instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id='') except BadRequest as ex: self.fail("failed to create new InstrumentSite: %s" %ex) print 'test_deployAsPrimaryDevice: new instrumentSite id = ', instrumentSite_id self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id) #------------------------------- # Logical Transform: Output Data Products #------------------------------- # Construct temporal and spatial Coordinate Reference System objects tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id) raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.pubsubclient.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id) #------------------------------- # Create Old InstrumentDevice #------------------------------- instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDeviceYear1', description="SBE37IMDevice for the FIRST year of deployment", serial_number="12345" ) try: oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) print 'test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ', oldInstDevice_id self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='SiteDataProduct', description='SiteDataProduct', temporal_domain = tdom, spatial_domain = sdom) instrument_site_output_dp_id = self.dataproductclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) self.damsclient.assign_data_product(input_resource_id=oldInstDevice_id, data_product_id=instrument_site_output_dp_id) #self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) self.omsclient.create_site_data_product(instrumentSite_id, instrument_site_output_dp_id) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='first deployment') oldDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id) #------------------------------- # Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information # cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port, #------------------------------- port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear1', description="SBE37IMAgentInstanceYear1", port_agent_config = port_agent_config, stream_configurations = [raw_config, parsed_config]) oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, oldInstDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() #------------------------------- # Create CTD Parsed as the Year 1 data product and attach to instrument #------------------------------- print 'Creating new CDM data product with a stream definition' dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year1', description='ctd stream test year 1', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year1 self.damsclient.assign_data_product(input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams1 = ', stream_ids #------------------------------- # Create New InstrumentDevice #------------------------------- instDevice_obj_2 = IonObject(RT.InstrumentDevice, name='SBE37IMDeviceYear2', description="SBE37IMDevice for the SECOND year of deployment", serial_number="67890" ) try: newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) print 'test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ', newInstDevice_id #set the LCSTATE self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY) self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE) instDevice_obj_2 = self.rrclient.read(newInstDevice_id) log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate)) #------------------------------- # Create Old Deployment #------------------------------- deployment_obj = IonObject(RT.Deployment, name='second deployment') newDeployment_id = self.omsclient.create_deployment(deployment_obj) # deploy this device to the logical slot self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id) self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id) #------------------------------- # Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information #------------------------------- port_agent_config = { 'device_addr': 'sbe37-simulator.oceanobservatories.org', 'device_port': 4004, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': 4005, 'data_port': 4006, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstanceYear2', description="SBE37IMAgentInstanceYear2", port_agent_config = port_agent_config) newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, newInstDevice_id) #------------------------------- # Create CTD Parsed as the Year 2 data product #------------------------------- dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_year2', description='ctd stream test year 2', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year2 self.damsclient.assign_data_product(input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True) print 'test_deployAsPrimaryDevice: Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' ) outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' ) outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=parsed_pdict_id) self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' ) self.output_products={} log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(data_product=ctd_l0_conductivity_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(data_product=ctd_l0_pressure_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(data_product=ctd_l0_temperature_output_dp_obj, stream_definition_id=parsed_stream_def_id) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id #self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product) #------------------------------- log.debug("test_deployAsPrimaryDevice: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_deployAsPrimaryDevice: create L0 all data_process return") #-------------------------------- # Activate the deployment #-------------------------------- self.omsclient.activate_deployment(oldDeployment_id) #------------------------------- # Launch InstrumentAgentInstance Sim1, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=oldInstAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) inst_agent1_instance_obj= self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent1_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim1 = ResourceAgentClient('iaclient Sim1', name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim1 %s', self._ia_client_sim1 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1)) #------------------------------- # Launch InstrumentAgentInstance Sim2, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id) self.addCleanup(self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=newInstAgentInstance_id) #wait for start instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) gate = ProcessStateGate(self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) inst_agent2_instance_obj= self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id) print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent2_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client_sim2 = ResourceAgentClient('iaclient Sim2', name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess()) print 'activate_instrument: got _ia_client_sim2 %s', self._ia_client_sim2 log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2)) #------------------------------- # Streaming Sim1 (old instrument) #------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active %s", str(reply)) self.assertTrue(reply) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim1.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE: %s", str(retval)) #------------------------------- # Streaming Sim 2 (new instrument) #------------------------------- cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) retval = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s", str(retval)) log.debug("(L4-CI-SA-RQ-334): Sending go_active command ") cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: return value from go_active_sim2 %s", str(reply)) cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE) retval = self._ia_client_sim2.execute_agent(cmd) state = retval.result log.debug("(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s", str(state)) cmd = AgentCommand(command=ResourceAgentEvent.RUN) reply = self._ia_client_sim2.execute_agent(cmd) log.debug("test_deployAsPrimaryDevice: run %s", str(reply)) gevent.sleep(2) cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s", str(retval)) gevent.sleep(10) #------------------------------- # Shutdown Sim1 (old instrument) #------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim1.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset %s", str(reply)) time.sleep(5) #------------------------------- # Shutdown Sim2 (old instrument) #------------------------------- cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE) retval = self._ia_client_sim2.execute_resource(cmd) log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s", str(retval)) log.debug("test_activateInstrumentSample: calling reset_sim2 ") cmd = AgentCommand(command=ResourceAgentEvent.RESET) reply = self._ia_client_sim1.execute_agent(cmd) log.debug("test_activateInstrumentSample: return from reset_sim2 %s", str(reply)) time.sleep(5)
def test_resource_state_save_restore(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel") instModel_id = self.IMS.create_instrument_model(instModel_obj) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 ) parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 ) instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg", stream_configurations = [raw_config, parsed_config] ) instAgent_id = self.IMS.create_instrument_agent(instAgent_obj) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 ' + '(SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj) self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) port_agent_config = { 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'process_type': PortAgentProcessType.UNIX, 'binary_path': "port_agent", 'port_agent_addr': 'localhost', 'command_port': CFG.device.sbe37.port_agent_cmd_port, 'data_port': CFG.device.sbe37.port_agent_data_port, 'log_level': 5, 'type': PortAgentType.ETHERNET } instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", port_agent_config = port_agent_config) instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id) rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True) raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id) #------------------------------- # Create Raw and Parsed Data Products for the device #------------------------------- dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id) log.debug( 'new dp_id = %s', data_product_id1) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.DP.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) # Retrieve the id of the OUTPUT stream from the out Data Product dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True) log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0]) self.parsed_dataset = dataset_ids[0] #create the datastore at the beginning of each int test that persists data dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) data_product_id2 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id) log.debug( 'new dp_id = %s', str(data_product_id2)) self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.DP.activate_data_product_persistence(data_product_id=data_product_id2) # spin up agent self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) self.addCleanup(self.IMS.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id) #wait for start instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id) gate = ProcessStateGate(self.PDC.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING) self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id) # take snapshot of config snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot") snap_obj = self.RR.read_attachment(snap_id, include_content=True) print "Saved config:" print snap_obj.content #modify config instance_obj.driver_config["comms_config"] = "BAD_DATA" self.RR.update(instance_obj) #restore config self.IMS.restore_resource_state(instDevice_id, snap_id) instance_obj = self.RR.read(instAgentInstance_id) self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"]) self.DP.delete_data_product(data_product_id1) self.DP.delete_data_product(data_product_id2)