def test_sample(self): result = yield self.driver_client.initialize('some arg') dpsc = DataPubsubClient(self.sup) topicname = 'SBE49 Topic' topic = PubSubTopicResource.create(topicname, "") # Use the service to create a queue and register the topic topic = yield dpsc.define_topic(topic) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create(topicname, '') subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } subscription = yield dpsc.define_subscription(subscription) logging.info('Defined subscription: ' + str(subscription)) params = {} params['publish-to'] = topic.RegistryIdentity yield self.driver_client.configure_driver(params) cmd1 = [['ds', 'now']] result = yield self.driver_client.execute(cmd1) self.assertEqual(result['status'], 'OK') yield pu.asleep(1) result = yield self.driver_client.disconnect(['some arg'])
def test_sample(self): result = yield self.driver_client.initialize('some arg') dpsc = DataPubsubClient(self.sup) topicname = 'SBE49 Topic' topic = PubSubTopicResource.create(topicname,"") # Use the service to create a queue and register the topic topic = yield dpsc.define_topic(topic) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create(topicname,'') subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } subscription = yield dpsc.define_subscription(subscription) logging.info('Defined subscription: '+str(subscription)) params = {} params['publish-to'] = topic.RegistryIdentity yield self.driver_client.configure_driver(params) cmd1 = [['ds', 'now']] result = yield self.driver_client.execute(cmd1) self.assertEqual(result['status'], 'OK') yield pu.asleep(1) result = yield self.driver_client.disconnect(['some arg'])
def test_create_topic(self): #dpsc = DataPubsubClient(self.pubsubSuper) dpsc = DataPubsubClient(self.sup) # Create and Register a topic """ DHE: not sure the driver should be creating the topic; for right now I'll have the test case do it. """ self.topic = PubSubTopicResource.create('SBE49 Topic',"oceans, oil spill") self.topic = yield dpsc.define_topic(self.topic) print 'TADA!'
def test_create_topic(self): #dpsc = DataPubsubClient(self.pubsubSuper) dpsc = DataPubsubClient(self.sup) # Create and Register a topic """ DHE: not sure the driver should be creating the topic; for right now I'll have the test case do it. """ self.topic = PubSubTopicResource.create('SBE49 Topic', "oceans, oil spill") self.topic = yield dpsc.define_topic(self.topic) print 'TADA!'
def _configure_driver(self, params): """ Configures driver params either on startup or on command """ if 'publish-to' in params: self.publish_to = params['publish-to'] logging.debug("Configured publish-to=" + self.publish_to) self.topicDefined = True self.dpsc = DataPubsubClient(proc=self) self.topic = ResourceReference(RegistryIdentity=self.publish_to, RegistryBranch='master') self.publisher = PublisherResource.create('Test Publisher', self, self.topic, 'DataObject') self.publisher = yield self.dpsc.define_publisher(self.publisher)
def test_execute(self): """ Test the execute command to the Instrument Driver """ result = yield self.driver_client.initialize('some arg') dpsc = DataPubsubClient(self.sup) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create('SBE49 Topic','') #subscription.topic2 = PubSubTopicResource.create('','oceans') subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } subscription = yield dpsc.define_subscription(subscription) logging.info('Defined subscription: '+str(subscription)) cmd1 = [['ds', 'now']] #cmd1 = [['start', 'now']] #cmd2 = [['stop', 'now']] #cmd2 = [['pumpoff', '3600', '1']] result = yield self.driver_client.execute(cmd1) self.assertEqual(result['status'], 'OK') # DHE: wait a while... yield pu.asleep(1) #result = yield self.driver_client.execute(cmd2) #self.assertEqual(result['status'], 'OK') # DHE: disconnecting; a connect would probably be good. result = yield self.driver_client.disconnect(['some arg'])
def test_execute(self): """ Test the execute command to the Instrument Driver """ result = yield self.driver_client.initialize('some arg') dpsc = DataPubsubClient(self.sup) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create('SBE49 Topic', '') #subscription.topic2 = PubSubTopicResource.create('','oceans') subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } subscription = yield dpsc.define_subscription(subscription) logging.info('Defined subscription: ' + str(subscription)) cmd1 = [['ds', 'now']] #cmd1 = [['start', 'now']] #cmd2 = [['stop', 'now']] #cmd2 = [['pumpoff', '3600', '1']] result = yield self.driver_client.execute(cmd1) self.assertEqual(result['status'], 'OK') # DHE: wait a while... yield pu.asleep(1) #result = yield self.driver_client.execute(cmd2) #self.assertEqual(result['status'], 'OK') # DHE: disconnecting; a connect would probably be good. result = yield self.driver_client.disconnect(['some arg'])
def slc_init(self): self.irc = InstrumentRegistryClient(proc=self) self.dprc = DataProductRegistryClient(proc=self) self.arc = AgentRegistryClient(proc=self) self.dpsc = DataPubsubClient(proc=self)
class InstrumentManagementService(BaseService): """ Instrument management service interface. This service provides overall coordination for instrument management within an observatory context. In particular it coordinates the access to the instrument and data product registries and the interaction with instrument agents. """ # Declaration of service declare = BaseService.service_declare(name='instrument_management', version='0.1.0', dependencies=[]) def slc_init(self): self.irc = InstrumentRegistryClient(proc=self) self.dprc = DataProductRegistryClient(proc=self) self.arc = AgentRegistryClient(proc=self) self.dpsc = DataPubsubClient(proc=self) @defer.inlineCallbacks def op_create_new_instrument(self, content, headers, msg): """ Service operation: Accepts a dictionary containing user inputs. Updates the instrument registry. """ userInput = content['userInput'] newinstrument = InstrumentResource.create_new_resource() if 'name' in userInput: newinstrument.name = str(userInput['name']) if 'description' in userInput: newinstrument.description = str(userInput['description']) if 'manufacturer' in userInput: newinstrument.manufacturer = str(userInput['manufacturer']) if 'model' in userInput: newinstrument.model = str(userInput['model']) if 'serial_num' in userInput: newinstrument.serial_num = str(userInput['serial_num']) if 'fw_version' in userInput: newinstrument.fw_version = str(userInput['fw_version']) instrument_res = yield self.irc.register_instrument_instance( newinstrument) yield self.reply_ok(msg, instrument_res.encode()) @defer.inlineCallbacks def op_create_new_data_product(self, content, headers, msg): """ Service operation: Accepts a dictionary containing user inputs. Updates the data product registry. Also sets up an ingestion pipeline for an instrument """ dataProductInput = content['dataProductInput'] newdp = DataProductResource.create_new_resource() if 'instrumentID' in dataProductInput: inst_id = str(dataProductInput['instrumentID']) int_ref = ResourceReference(RegistryIdentity=inst_id, RegistryBranch='master') newdp.instrument_ref = int_ref if 'name' in dataProductInput: newdp.name = str(dataProductInput['name']) if 'description' in dataProductInput: newdp.description = str(dataProductInput['description']) if 'dataformat' in dataProductInput: newdp.dataformat = str(dataProductInput['dataformat']) # Step: Create a data stream ## Instantiate a pubsubclient #self.dpsc = DataPubsubClient(proc=self) # ## Create and Register a topic #self.topic = PubSubTopicResource.create('SBE49 Topic',"oceans, oil spill") #self.topic = yield self.dpsc.define_topic(self.topic) #logging.debug('DHE: Defined Topic') # #self.publisher = PublisherResource.create('Test Publisher', self, self.topic, 'DataObject') #self.publisher = yield self.dpsc.define_publisher(self.publisher) res = yield self.dprc.register_data_product(newdp) ref = res.reference(head=True) yield self.reply_ok(msg, res.encode()) @defer.inlineCallbacks def op_execute_command(self, content, headers, msg): """ Service operation: Execute a command on an instrument. """ # Step 1: Extract the arguments from the UI generated message content commandInput = content['commandInput'] if 'instrumentID' in commandInput: inst_id = str(commandInput['instrumentID']) else: raise ValueError("Input for instrumentID not present") command = [] if 'command' in commandInput: command_op = str(commandInput['command']) else: raise ValueError("Input for command not present") command.append(command_op) arg_idx = 0 while True: argname = 'cmdArg' + str(arg_idx) arg_idx += 1 if argname in commandInput: command.append(str(commandInput[argname])) else: break # Step 2: Find the agent id for the given instrument id agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if not agent_pid: yield self.reply_err( msg, "No agent found for instrument " + str(inst_id)) defer.returnValue(None) # Step 3: Interact with the agent to execute the command iaclient = InstrumentAgentClient(proc=self, target=agent_pid) commandlist = [ command, ] logging.info("Sending command to IA: " + str(commandlist)) cmd_result = yield iaclient.execute_instrument(commandlist) yield self.reply_ok(msg, cmd_result) @defer.inlineCallbacks def op_get_instrument_state(self, content, headers, msg): """ Service operation: . """ # Step 1: Extract the arguments from the UI generated message content commandInput = content['commandInput'] if 'instrumentID' in commandInput: inst_id = str(commandInput['instrumentID']) else: raise ValueError("Input for instrumentID not present") agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if not agent_pid: raise StandardError("No agent found for instrument " + str(inst_id)) iaclient = InstrumentAgentClient(proc=self, target=agent_pid) inst_cap = yield iaclient.get_capabilities() if not inst_cap: raise StandardError("No capabilities available for instrument " + str(inst_id)) ci_commands = inst_cap['ci_commands'] instrument_commands = inst_cap['instrument_commands'] instrument_parameters = inst_cap['instrument_parameters'] ci_parameters = inst_cap['ci_parameters'] values = yield iaclient.get_from_instrument(instrument_parameters) resvalues = {} if values: resvalues = values yield self.reply_ok(msg, resvalues) @defer.inlineCallbacks def op_start_instrument_agent(self, content, headers, msg): """ Service operation: Starts an instrument agent for a type of instrument. """ if 'instrumentID' in content: inst_id = str(content['instrumentID']) else: raise ValueError("Input for instrumentID not present") if 'model' in content: model = str(content['model']) else: raise ValueError("Input for model not present") if model != 'SBE49': raise ValueError("Only SBE49 supported!") agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if agent_pid: raise StandardError("Agent already started for instrument " + str(inst_id)) simulator = Simulator(inst_id) simulator.start() topicname = "Inst/RAW/" + inst_id topic = PubSubTopicResource.create(topicname, "") # Use the service to create a queue and register the topic topic = yield self.dpsc.define_topic(topic) iagent_args = {} iagent_args['instrument-id'] = inst_id driver_args = {} driver_args['port'] = simulator.port driver_args['publish-to'] = topic.RegistryIdentity iagent_args['driver-args'] = driver_args iapd = ProcessDesc( **{ 'name': 'SBE49IA', 'module': 'ion.agents.instrumentagents.SBE49_IA', 'class': 'SBE49InstrumentAgent', 'spawnargs': iagent_args }) iagent_id = yield self.spawn_child(iapd) iaclient = InstrumentAgentClient(proc=self, target=iagent_id) yield iaclient.register_resource(inst_id) yield self.reply_ok(msg, "OK") @defer.inlineCallbacks def op_stop_instrument_agent(self, content, headers, msg): """ Service operation: Starts direct access mode. """ yield self.reply_err(msg, "Not yet implemented") @defer.inlineCallbacks def op_start_direct_access(self, content, headers, msg): """ Service operation: Starts direct access mode. """ yield self.reply_err(msg, "Not yet implemented") @defer.inlineCallbacks def op_stop_direct_access(self, content, headers, msg): """ Service operation: Stops direct access mode. """ yield self.reply_err(msg, "Not yet implemented") @defer.inlineCallbacks def get_agent_desc_for_instrument(self, instrument_id): logging.info("get_agent_desc_for_instrument() instrumentID=" + str(instrument_id)) int_ref = ResourceReference(RegistryIdentity=instrument_id, RegistryBranch='master') agent_query = InstrumentAgentResourceInstance() agent_query.instrument_ref = int_ref if not agent_res: defer.returnValue(None) agent_pid = agent_res.proc_id logging.info("Agent process id for instrument id %s is: %s" % (instrument_id, agent_pid)) defer.returnValue(agent_pid) @defer.inlineCallbacks def get_agent_for_instrument(self, instrument_id): logging.info("get_agent_for_instrument() instrumentID=" + str(instrument_id)) int_ref = ResourceReference(RegistryIdentity=instrument_id, RegistryBranch='master') agent_query = InstrumentAgentResourceInstance() agent_query.instrument_ref = int_ref # @todo Need to list the LC state here. WHY??? agent_query.lifecycle = LCStates.developed agents = yield self.arc.find_registered_agent_instance_from_description( agent_query, regex=False) logging.info("Found %s agent instances for instrument id %s" % (len(agents), instrument_id)) agent_res = None if len(agents) > 0: agent_res = agents[0] defer.returnValue(agent_res) @defer.inlineCallbacks def get_agent_pid_for_instrument(self, instrument_id): agent_res = yield self.get_agent_for_instrument(instrument_id) if not agent_res: defer.returnValue(None) agent_pid = agent_res.proc_id logging.info("Agent process id for instrument id %s is: %s" % (instrument_id, agent_pid)) defer.returnValue(agent_pid)
class SBE49InstrumentDriver(InstrumentDriver): """ Maybe some day these values are looked up from a registry of common controlled vocabulary """ def __init__(self, receiver=None, spawnArgs=None, **kwargs): self.connected = False self.instrument = None self.command = None self.topicDefined = False self.publish_to = None """ A translation dictionary to translate from the commands being sent from the agent to the actual command understood by the instrument. """ self.sbeParmCommands = { "baudrate" : "Baud", "outputformat" : "outputformat" } self.__instrument_parameters = { "baudrate": 9600, "outputformat": 0, "outputsal": "Y", "outputsv": "Y", "navg": 0, "mincondfreq": 0, "pumpdelay": 0, "tadvance": 0.0625, "alpha": 0.03, "tau": 7.0, "autorun": "Y", "tcaldate": "1/1/01", "ta0": 0.0, "ta1": 0.0, "ta2": 0.0, "ta3": 0.0, "toffset": 0.0, "ccaldate": "1/1/01", "cg": 0.0, "ch": 0.0, "ci": 0.0, "cj": 0.0, "cpcor": 0.0, "ctcor": 0.0, "cslope": 0.0, "pcaldate": "1/1/01", "prange": 100.0, "poffset": 0.0, "pa0": 0.0, "pa1": 0.0, "pa2": 0.0, "ptempa0": 0.0, "ptempa1": 0.0, "ptempa2": 0.0, "ptca0": 0.0, "ptca1": 0.0, "ptca2": 0.0, "ptcb0": 0.0, "ptcb1": 0.0, "ptcb2": 0.0 } InstrumentDriver.__init__(self, receiver, spawnArgs, **kwargs) @defer.inlineCallbacks def plc_init(self): self.instrument_id = self.spawn_args.get('instrument-id', '123') self.instrument_port = self.spawn_args.get('port', 9000) yield self._configure_driver(self.spawn_args) logging.info("INIT DRIVER for instrument ID=%s, port=%s, publish-to=%s" % ( self.instrument_id, self.instrument_port, self.publish_to)) self.iaclient = InstrumentAgentClient(proc=self, target=self.proc_supid) logging.debug("Instrument driver initialized") @defer.inlineCallbacks def plc_shutdown(self): yield self.op_disconnect(None, None, None) def isConnected(self): return self.connected def setConnected(self, value): self.connected = value; def setAgentService(self, agent): self.agent = agent @defer.inlineCallbacks def getConnected(self): """ @brief A method to get connected to the instrument device server. Right now this assumes the device is connected via a TCP/IP device server. We probably need to come up with a more flexible way of doing this; like getting a connection object that abstracts the details of the protocol. Not sure how easy that would be with Twisted and Python. Gets a deferred object passes it to the InstrumentClientFactory, which uses it to acess callbacks. Was trying to use this to make the connection process more managable. Not sure if that's the case or not yet. @retval The deferred object. """ # Now thinking I might try clientcreator since this will only be a # single connection. cc = ClientCreator(reactor, InstrumentClient, self) self.proto = yield cc.connectTCP("localhost", self.instrument_port) logging.info("Driver connected to instrument") def gotConnected(self, instrument): """ @brief This method is called when a connection has been made to the instrument device server. The instrument protocol object is passed as a parameter, and a reference to it is saved. Call setConnected with True argument. @param reference to instrument protocol object. @retval none """ logging.debug("gotConnected!!!") self.instrument = instrument self.setConnected(True) def gotDisconnected(self, instrument): """ @brief This method is called when a connection to the instrument device server has been lost. The instrument protocol object is passed as a parameter. Call setConnected with False argument. @param reference to instrument protocol object. @retval none """ logging.debug("gotDisconnected!!!") self.instrument = instrument self.setConnected(False) def gotData(self, data): """ @brief The instrument protocol object has received data from the instrument. It should already be sanitized and ready for consumption; publish the data. @param data @retval none """ # send this up to the agent to publish. logging.debug("gotData() %s Calling publish." % (data)) self.publish(data, self.publish_to) def gotPrompt(self, instrument): """ This needs to be the general receive routine for the instrument driver """ logging.debug("gotPrompt()") #self.instrument = instrument #self.setConnected(True) """ Do we need some sort of state machine so we'll know what data we're supposed to send here? Right now it's working without... """ @defer.inlineCallbacks def publish(self, data, topic): """ @Brief Publish the given data to the given topic. @param data The data to publish @param topic The topic to which to publish. Currently this is not the topic as defined by pubsub. @retval none """ logging.debug("publish()") if self.topicDefined == True: # Create and send a data message result = yield self.dpsc.publish(self, self.topic.reference(), data) if result: logging.info('Published Message') else: logging.info('Failed to Published Message') else: logging.info("NOT READY TO PUBLISH") @defer.inlineCallbacks def op_initialize(self, content, headers, msg): logging.debug('In driver initialize') yield self.reply_ok(msg, content) @defer.inlineCallbacks def op_disconnect(self, content, headers, msg): logging.debug("in Instrument Driver op_disconnect!") if (self.isConnected()): logging.debug("disconnecting from instrument") #self.connector.disconnect() self.proto.transport.loseConnection() self.setConnected(False) if msg: yield self.reply_ok(msg, content) @defer.inlineCallbacks def op_fetch_params(self, content, headers, msg): """ Operate in instrument protocol to get parameter @todo Write the code to interface this with something """ assert(isinstance(content, (list, tuple))) result = {} for param in content: result[param] = self.__instrument_parameters[param] yield self.reply_ok(msg, result) @defer.inlineCallbacks def op_set_params(self, content, headers, msg): """ Operate in instrument protocol to set a parameter. Current semantics are that, if there is a problem, fail as soon as possible. This may leave partial settings made in the device. @param content A dict of all the parameters and values to set @todo Make this an all-or-nothing and/or rollback-able transaction list? """ """ This connection stuff could be abstracted into a communications object. """ if self.isConnected() == False: logging.debug("yielding for connect") yield self.getConnected() logging.debug("connect returned") assert(isinstance(content, dict)) logging.debug("op_set_params content: %s, keys: %s" %(str(content), str(content.keys))) for param in content.keys(): if (param not in self.__instrument_parameters): yield self.reply_err(msg, "Could not set %s" % param) else: self.__instrument_parameters[param] = content[param] if param in self.sbeParmCommands: if self.isConnected(): logging.info("current param is: %s" %str(param)) command = self.sbeParmCommands[param] + "=" + str(content[param]) logging.debug("op_set_params sending %s to instrument" %str(command)) self.instrument.transport.write(command) else: logging.error("%s is not a settable parameter" % str(param)) yield self.reply_ok(msg, content) @defer.inlineCallbacks def op_execute(self, content, headers, msg): """ Execute the given command structure (first element command, rest of the elements are arguments) @todo actually do something """ assert(isinstance(content, (tuple, list))) logging.debug("op_execute content: %s" %str(content)) """ This connection stuff could be abstracted into a communications object. """ if self.isConnected() == False: logging.info("yielding for connect") yield self.getConnected() logging.info("connect returned") # DHE NOTE TO SELF: not using the addCallback anymore, but it might # be a good way to implement a state machine. #d.addCallback(self.gotConnected); #d.addCallback(self.gotPrompt); if ((content == ()) or (content == [])): yield self.reply_err(msg, "Empty command") return commands = [] for command_set in content: command = command_set[0] if command not in instrument_commands: logging.error("Invalid Command") yield self.reply_err(msg, "Invalid Command") else: logging.debug("op_execute sending command: %s to instrument" % command) self.command = command """ Currently sending the command from right here. We SHOULD be connected at this point. """ if self.isConnected(): self.instrument.transport.write(self.command) else: logging.error("op_execute: instrument not connected.") commands.append(command) yield self.reply_ok(msg, commands) @defer.inlineCallbacks def op_get_status(self, content, headers, msg): """ Return the non-parameter and non-lifecycle status of the instrument. This may include a snippit of config or important summary of what the instrument may be doing...or even something else. @param args a list of arguments that may be given for the status retreival. @return Return a tuple of (status_code, dict) @todo Remove this? Is it even used? """ yield self.reply_ok(msg, "a-ok") @defer.inlineCallbacks def op_configure_driver(self, content, headers, msg): """ This method takes a dict of settings that the driver understands as configuration of the driver itself (ie 'target_ip', 'port', etc.). This is the bootstrap information for the driver and includes enough information for the driver to start communicating with the instrument. @param content A dict with parameters for the driver @todo Actually make this stub do something """ assert(isinstance(content, dict)) yield self._configure_driver(content) # Do something here, then adjust test case yield self.reply_ok(msg, content) @defer.inlineCallbacks def _configure_driver(self, params): """ Configures driver params either on startup or on command """ if 'publish-to' in params: self.publish_to = params['publish-to'] logging.debug("Configured publish-to=" + self.publish_to) self.topicDefined = True self.dpsc = DataPubsubClient(proc=self) self.topic = ResourceReference(RegistryIdentity=self.publish_to, RegistryBranch='master') self.publisher = PublisherResource.create('Test Publisher', self, self.topic, 'DataObject') self.publisher = yield self.dpsc.define_publisher(self.publisher)
def test_exampleconsumer(self): ''' @Brief Example Consumer is a demonstration of a more complex data consumer. It uses DAP data messages and provides qaqc and event results on two seperate queues. ''' dpsc = DataPubsubClient(self.sup) #Create and register 3 topics! topic_raw = PubSubTopicResource.create("topic_raw","oceans, oil spill, fun things to do") topic_raw = yield dpsc.define_topic(topic_raw) #Create and register self.sup as a publisher publisher = PublisherResource.create('Test Publisher', self.sup, topic_raw, 'DataObject') publisher = yield dpsc.define_publisher(publisher) logging.info('Defined Publisher: '+str(publisher)) # === Create a Consumer and queues - this will become part of define_subscription. #Create two test queues - don't use topics to test the consumer # To be replaced when the subscription service is ready evt_queue=dataobject.create_unique_identity() queue_properties = {evt_queue:{'name_type':'fanout', 'args':{'scope':'global'}}} yield bootstrap.declare_messaging(queue_properties) pr_queue=dataobject.create_unique_identity() queue_properties = {pr_queue:{'name_type':'fanout', 'args':{'scope':'global'}}} yield bootstrap.declare_messaging(queue_properties) pd1={'name':'example_consumer_1', 'module':'ion.services.dm.distribution.consumers.example_consumer', 'procclass':'ExampleConsumer', 'spawnargs':{'attach':topic_raw.queue.name,\ 'Process Parameters':{},\ 'delivery queues':\ {'event_queue':evt_queue,\ 'processed_queue':pr_queue}}\ } child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) pd2={'name':'example_consumer_2', 'module':'ion.services.dm.distribution.consumers.logging_consumer', 'procclass':'LoggingConsumer', 'spawnargs':{'attach':evt_queue,\ 'Process Parameters':{}}\ } child2 = base_consumer.ConsumerDesc(**pd2) child2_id = yield self.test_sup.spawn_child(child2) pd3={'name':'example_consumer_3', 'module':'ion.services.dm.distribution.consumers.logging_consumer', 'procclass':'LoggingConsumer', 'spawnargs':{'attach':pr_queue,\ 'Process Parameters':{}}\ } child3 = base_consumer.ConsumerDesc(**pd3) child3_id = yield self.test_sup.spawn_child(child3) # === End of stuff that will be replaced with Subscription method... # Create an example data message dmsg = dap_tools.simple_datamessage(\ {'DataSet Name':'Simple Data','variables':\ {'time':{'long_name':'Data and Time','units':'seconds'},\ 'height':{'long_name':'person height','units':'meters'}}}, \ {'time':(101,102,103,104,105,106,107,108,109,110), \ 'height':(5,2,4,5,-1,9,3,888,3,4)}) result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg) if result: logging.info('Published Message') else: logging.info('Failed to Published Message') # Need to await the delivery of data messages into the consumers yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(evt_queue),2) self.assertEqual(sent.get(pr_queue),1) self.assertEqual(received.get(topic_raw.queue.name),1) msg_cnt = yield child2.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(evt_queue),2) msg_cnt = yield child3.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(pr_queue),1) # Publish a second message with different data dmsg = dap_tools.simple_datamessage(\ {'DataSet Name':'Simple Data','variables':\ {'time':{'long_name':'Data and Time','units':'seconds'},\ 'height':{'long_name':'person height','units':'meters'}}}, \ {'time':(111,112,123,114,115,116,117,118,119,120), \ 'height':(8,986,4,-2,-1,5,3,1,4,5)}) result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg) # Need to await the delivery of data messages into the consumers yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(evt_queue),5) self.assertEqual(sent.get(pr_queue),2) self.assertEqual(received.get(topic_raw.queue.name),2) msg_cnt = yield child2.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(evt_queue),5) msg_cnt = yield child3.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(pr_queue),2)
def test_pubsub(self): dpsc = DataPubsubClient(self.sup) # Create and Register a topic topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do") topic = yield dpsc.define_topic(topic) logging.info('Defined Topic: '+str(topic)) #Create and register self.sup as a publisher publisher = PublisherResource.create('Test Publisher', self.sup, topic, 'DataObject') publisher = yield dpsc.define_publisher(publisher) logging.info('Defined Publisher: '+str(publisher)) # === Create a Consumer and queues - this will become part of define_subscription. #Create two test queues - don't use topics to test the consumer # To be replaced when the subscription service is ready queue1=dataobject.create_unique_identity() queue_properties = {queue1:{'name_type':'fanout', 'args':{'scope':'global'}}} yield bootstrap.declare_messaging(queue_properties) queue2=dataobject.create_unique_identity() queue_properties = {queue2:{'name_type':'fanout', 'args':{'scope':'global'}}} yield bootstrap.declare_messaging(queue_properties) pd1={'name':'example_consumer_1', 'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'procclass':'ForwardingConsumer', 'spawnargs':{'attach':topic.queue.name,\ 'process parameters':{},\ 'delivery queues':{'queues':[queue1,queue2]}}\ } child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) # === End to be replaces with Define_Consumer # Create and send a data message data = {'Data':'in a dictionary'} result = yield dpsc.publish(self.sup, topic.reference(), data) if result: logging.info('Published Message') else: logging.info('Failed to Published Message') # Need to await the delivery of data messages into the (separate) consumers yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(queue1),1) self.assertEqual(sent.get(queue2),1) self.assertEqual(received.get(topic.queue.name),1) # === Create a Consumer - this will become part of define_subscription. pd2={'name':'example_consumer_2', 'module':'ion.services.dm.distribution.consumers.logging_consumer', 'procclass':'LoggingConsumer', 'spawnargs':{'attach':queue1,\ 'process parameters':{},\ 'delivery queues':{}}\ } child2 = base_consumer.ConsumerDesc(**pd2) child2_id = yield self.test_sup.spawn_child(child2) # === End of what will become part of the subscription definition # Send the simple message again result = yield dpsc.publish(self.sup, topic.reference(), data) # Need to await the delivery of data messages into the (separate) consumers yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(queue1),2) self.assertEqual(sent.get(queue2),2) self.assertEqual(received.get(topic.queue.name),2) msg_cnt = yield child2.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(queue1),1)
class SBE49InstrumentDriver(InstrumentDriver): """ Maybe some day these values are looked up from a registry of common controlled vocabulary """ def __init__(self, receiver=None, spawnArgs=None, **kwargs): self.connected = False self.instrument = None self.command = None self.topicDefined = False self.publish_to = None """ A translation dictionary to translate from the commands being sent from the agent to the actual command understood by the instrument. """ self.sbeParmCommands = { "baudrate": "Baud", "outputformat": "outputformat" } self.__instrument_parameters = { "baudrate": 9600, "outputformat": 0, "outputsal": "Y", "outputsv": "Y", "navg": 0, "mincondfreq": 0, "pumpdelay": 0, "tadvance": 0.0625, "alpha": 0.03, "tau": 7.0, "autorun": "Y", "tcaldate": "1/1/01", "ta0": 0.0, "ta1": 0.0, "ta2": 0.0, "ta3": 0.0, "toffset": 0.0, "ccaldate": "1/1/01", "cg": 0.0, "ch": 0.0, "ci": 0.0, "cj": 0.0, "cpcor": 0.0, "ctcor": 0.0, "cslope": 0.0, "pcaldate": "1/1/01", "prange": 100.0, "poffset": 0.0, "pa0": 0.0, "pa1": 0.0, "pa2": 0.0, "ptempa0": 0.0, "ptempa1": 0.0, "ptempa2": 0.0, "ptca0": 0.0, "ptca1": 0.0, "ptca2": 0.0, "ptcb0": 0.0, "ptcb1": 0.0, "ptcb2": 0.0 } InstrumentDriver.__init__(self, receiver, spawnArgs, **kwargs) @defer.inlineCallbacks def plc_init(self): self.instrument_id = self.spawn_args.get('instrument-id', '123') self.instrument_port = self.spawn_args.get('port', 9000) yield self._configure_driver(self.spawn_args) logging.info( "INIT DRIVER for instrument ID=%s, port=%s, publish-to=%s" % (self.instrument_id, self.instrument_port, self.publish_to)) self.iaclient = InstrumentAgentClient(proc=self, target=self.proc_supid) logging.debug("Instrument driver initialized") @defer.inlineCallbacks def plc_shutdown(self): yield self.op_disconnect(None, None, None) def isConnected(self): return self.connected def setConnected(self, value): self.connected = value def setAgentService(self, agent): self.agent = agent @defer.inlineCallbacks def getConnected(self): """ @brief A method to get connected to the instrument device server. Right now this assumes the device is connected via a TCP/IP device server. We probably need to come up with a more flexible way of doing this; like getting a connection object that abstracts the details of the protocol. Not sure how easy that would be with Twisted and Python. Gets a deferred object passes it to the InstrumentClientFactory, which uses it to acess callbacks. Was trying to use this to make the connection process more managable. Not sure if that's the case or not yet. @retval The deferred object. """ # Now thinking I might try clientcreator since this will only be a # single connection. cc = ClientCreator(reactor, InstrumentClient, self) self.proto = yield cc.connectTCP("localhost", self.instrument_port) logging.info("Driver connected to instrument") def gotConnected(self, instrument): """ @brief This method is called when a connection has been made to the instrument device server. The instrument protocol object is passed as a parameter, and a reference to it is saved. Call setConnected with True argument. @param reference to instrument protocol object. @retval none """ logging.debug("gotConnected!!!") self.instrument = instrument self.setConnected(True) def gotDisconnected(self, instrument): """ @brief This method is called when a connection to the instrument device server has been lost. The instrument protocol object is passed as a parameter. Call setConnected with False argument. @param reference to instrument protocol object. @retval none """ logging.debug("gotDisconnected!!!") self.instrument = instrument self.setConnected(False) def gotData(self, data): """ @brief The instrument protocol object has received data from the instrument. It should already be sanitized and ready for consumption; publish the data. @param data @retval none """ # send this up to the agent to publish. logging.debug("gotData() %s Calling publish." % (data)) self.publish(data, self.publish_to) def gotPrompt(self, instrument): """ This needs to be the general receive routine for the instrument driver """ logging.debug("gotPrompt()") #self.instrument = instrument #self.setConnected(True) """ Do we need some sort of state machine so we'll know what data we're supposed to send here? Right now it's working without... """ @defer.inlineCallbacks def publish(self, data, topic): """ @Brief Publish the given data to the given topic. @param data The data to publish @param topic The topic to which to publish. Currently this is not the topic as defined by pubsub. @retval none """ logging.debug("publish()") if self.topicDefined == True: # Create and send a data message result = yield self.dpsc.publish(self, self.topic.reference(), data) if result: logging.info('Published Message') else: logging.info('Failed to Published Message') else: logging.info("NOT READY TO PUBLISH") @defer.inlineCallbacks def op_initialize(self, content, headers, msg): logging.debug('In driver initialize') yield self.reply_ok(msg, content) @defer.inlineCallbacks def op_disconnect(self, content, headers, msg): logging.debug("in Instrument Driver op_disconnect!") if (self.isConnected()): logging.debug("disconnecting from instrument") #self.connector.disconnect() self.proto.transport.loseConnection() self.setConnected(False) if msg: yield self.reply_ok(msg, content) @defer.inlineCallbacks def op_fetch_params(self, content, headers, msg): """ Operate in instrument protocol to get parameter @todo Write the code to interface this with something """ assert (isinstance(content, (list, tuple))) result = {} for param in content: result[param] = self.__instrument_parameters[param] yield self.reply_ok(msg, result) @defer.inlineCallbacks def op_set_params(self, content, headers, msg): """ Operate in instrument protocol to set a parameter. Current semantics are that, if there is a problem, fail as soon as possible. This may leave partial settings made in the device. @param content A dict of all the parameters and values to set @todo Make this an all-or-nothing and/or rollback-able transaction list? """ """ This connection stuff could be abstracted into a communications object. """ if self.isConnected() == False: logging.debug("yielding for connect") yield self.getConnected() logging.debug("connect returned") assert (isinstance(content, dict)) logging.debug("op_set_params content: %s, keys: %s" % (str(content), str(content.keys))) for param in content.keys(): if (param not in self.__instrument_parameters): yield self.reply_err(msg, "Could not set %s" % param) else: self.__instrument_parameters[param] = content[param] if param in self.sbeParmCommands: if self.isConnected(): logging.info("current param is: %s" % str(param)) command = self.sbeParmCommands[param] + "=" + str( content[param]) logging.debug( "op_set_params sending %s to instrument" % str(command)) self.instrument.transport.write(command) else: logging.error("%s is not a settable parameter" % str(param)) yield self.reply_ok(msg, content) @defer.inlineCallbacks def op_execute(self, content, headers, msg): """ Execute the given command structure (first element command, rest of the elements are arguments) @todo actually do something """ assert (isinstance(content, (tuple, list))) logging.debug("op_execute content: %s" % str(content)) """ This connection stuff could be abstracted into a communications object. """ if self.isConnected() == False: logging.info("yielding for connect") yield self.getConnected() logging.info("connect returned") # DHE NOTE TO SELF: not using the addCallback anymore, but it might # be a good way to implement a state machine. #d.addCallback(self.gotConnected); #d.addCallback(self.gotPrompt); if ((content == ()) or (content == [])): yield self.reply_err(msg, "Empty command") return commands = [] for command_set in content: command = command_set[0] if command not in instrument_commands: logging.error("Invalid Command") yield self.reply_err(msg, "Invalid Command") else: logging.debug("op_execute sending command: %s to instrument" % command) self.command = command """ Currently sending the command from right here. We SHOULD be connected at this point. """ if self.isConnected(): self.instrument.transport.write(self.command) else: logging.error("op_execute: instrument not connected.") commands.append(command) yield self.reply_ok(msg, commands) @defer.inlineCallbacks def op_get_status(self, content, headers, msg): """ Return the non-parameter and non-lifecycle status of the instrument. This may include a snippit of config or important summary of what the instrument may be doing...or even something else. @param args a list of arguments that may be given for the status retreival. @return Return a tuple of (status_code, dict) @todo Remove this? Is it even used? """ yield self.reply_ok(msg, "a-ok") @defer.inlineCallbacks def op_configure_driver(self, content, headers, msg): """ This method takes a dict of settings that the driver understands as configuration of the driver itself (ie 'target_ip', 'port', etc.). This is the bootstrap information for the driver and includes enough information for the driver to start communicating with the instrument. @param content A dict with parameters for the driver @todo Actually make this stub do something """ assert (isinstance(content, dict)) yield self._configure_driver(content) # Do something here, then adjust test case yield self.reply_ok(msg, content) @defer.inlineCallbacks def _configure_driver(self, params): """ Configures driver params either on startup or on command """ if 'publish-to' in params: self.publish_to = params['publish-to'] logging.debug("Configured publish-to=" + self.publish_to) self.topicDefined = True self.dpsc = DataPubsubClient(proc=self) self.topic = ResourceReference(RegistryIdentity=self.publish_to, RegistryBranch='master') self.publisher = PublisherResource.create('Test Publisher', self, self.topic, 'DataObject') self.publisher = yield self.dpsc.define_publisher(self.publisher)
class InstrumentManagementService(BaseService): """ Instrument management service interface. This service provides overall coordination for instrument management within an observatory context. In particular it coordinates the access to the instrument and data product registries and the interaction with instrument agents. """ # Declaration of service declare = BaseService.service_declare(name='instrument_management', version='0.1.0', dependencies=[]) def slc_init(self): self.irc = InstrumentRegistryClient(proc=self) self.dprc = DataProductRegistryClient(proc=self) self.arc = AgentRegistryClient(proc=self) self.dpsc = DataPubsubClient(proc=self) @defer.inlineCallbacks def op_create_new_instrument(self, content, headers, msg): """ Service operation: Accepts a dictionary containing user inputs. Updates the instrument registry. """ userInput = content['userInput'] newinstrument = InstrumentResource.create_new_resource() if 'name' in userInput: newinstrument.name = str(userInput['name']) if 'description' in userInput: newinstrument.description = str(userInput['description']) if 'manufacturer' in userInput: newinstrument.manufacturer = str(userInput['manufacturer']) if 'model' in userInput: newinstrument.model = str(userInput['model']) if 'serial_num' in userInput: newinstrument.serial_num = str(userInput['serial_num']) if 'fw_version' in userInput: newinstrument.fw_version = str(userInput['fw_version']) instrument_res = yield self.irc.register_instrument_instance(newinstrument) yield self.reply_ok(msg, instrument_res.encode()) @defer.inlineCallbacks def op_create_new_data_product(self, content, headers, msg): """ Service operation: Accepts a dictionary containing user inputs. Updates the data product registry. Also sets up an ingestion pipeline for an instrument """ dataProductInput = content['dataProductInput'] newdp = DataProductResource.create_new_resource() if 'instrumentID' in dataProductInput: inst_id = str(dataProductInput['instrumentID']) int_ref = ResourceReference(RegistryIdentity=inst_id, RegistryBranch='master') newdp.instrument_ref = int_ref if 'name' in dataProductInput: newdp.name = str(dataProductInput['name']) if 'description' in dataProductInput: newdp.description = str(dataProductInput['description']) if 'dataformat' in dataProductInput: newdp.dataformat = str(dataProductInput['dataformat']) # Step: Create a data stream ## Instantiate a pubsubclient #self.dpsc = DataPubsubClient(proc=self) # ## Create and Register a topic #self.topic = PubSubTopicResource.create('SBE49 Topic',"oceans, oil spill") #self.topic = yield self.dpsc.define_topic(self.topic) #logging.debug('DHE: Defined Topic') # #self.publisher = PublisherResource.create('Test Publisher', self, self.topic, 'DataObject') #self.publisher = yield self.dpsc.define_publisher(self.publisher) res = yield self.dprc.register_data_product(newdp) ref = res.reference(head=True) yield self.reply_ok(msg, res.encode()) @defer.inlineCallbacks def op_execute_command(self, content, headers, msg): """ Service operation: Execute a command on an instrument. """ # Step 1: Extract the arguments from the UI generated message content commandInput = content['commandInput'] if 'instrumentID' in commandInput: inst_id = str(commandInput['instrumentID']) else: raise ValueError("Input for instrumentID not present") command = [] if 'command' in commandInput: command_op = str(commandInput['command']) else: raise ValueError("Input for command not present") command.append(command_op) arg_idx = 0 while True: argname = 'cmdArg'+str(arg_idx) arg_idx += 1 if argname in commandInput: command.append(str(commandInput[argname])) else: break # Step 2: Find the agent id for the given instrument id agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if not agent_pid: yield self.reply_err(msg, "No agent found for instrument "+str(inst_id)) defer.returnValue(None) # Step 3: Interact with the agent to execute the command iaclient = InstrumentAgentClient(proc=self, target=agent_pid) commandlist = [command,] logging.info("Sending command to IA: "+str(commandlist)) cmd_result = yield iaclient.execute_instrument(commandlist) yield self.reply_ok(msg, cmd_result) @defer.inlineCallbacks def op_get_instrument_state(self, content, headers, msg): """ Service operation: . """ # Step 1: Extract the arguments from the UI generated message content commandInput = content['commandInput'] if 'instrumentID' in commandInput: inst_id = str(commandInput['instrumentID']) else: raise ValueError("Input for instrumentID not present") agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if not agent_pid: raise StandardError("No agent found for instrument "+str(inst_id)) iaclient = InstrumentAgentClient(proc=self, target=agent_pid) inst_cap = yield iaclient.get_capabilities() if not inst_cap: raise StandardError("No capabilities available for instrument "+str(inst_id)) ci_commands = inst_cap['ci_commands'] instrument_commands = inst_cap['instrument_commands'] instrument_parameters = inst_cap['instrument_parameters'] ci_parameters = inst_cap['ci_parameters'] values = yield iaclient.get_from_instrument(instrument_parameters) resvalues = {} if values: resvalues = values yield self.reply_ok(msg, resvalues) @defer.inlineCallbacks def op_start_instrument_agent(self, content, headers, msg): """ Service operation: Starts an instrument agent for a type of instrument. """ if 'instrumentID' in content: inst_id = str(content['instrumentID']) else: raise ValueError("Input for instrumentID not present") if 'model' in content: model = str(content['model']) else: raise ValueError("Input for model not present") if model != 'SBE49': raise ValueError("Only SBE49 supported!") agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if agent_pid: raise StandardError("Agent already started for instrument "+str(inst_id)) simulator = Simulator(inst_id) simulator.start() topicname = "Inst/RAW/"+inst_id topic = PubSubTopicResource.create(topicname,"") # Use the service to create a queue and register the topic topic = yield self.dpsc.define_topic(topic) iagent_args = {} iagent_args['instrument-id'] = inst_id driver_args = {} driver_args['port'] = simulator.port driver_args['publish-to'] = topic.RegistryIdentity iagent_args['driver-args'] = driver_args iapd = ProcessDesc(**{'name':'SBE49IA', 'module':'ion.agents.instrumentagents.SBE49_IA', 'class':'SBE49InstrumentAgent', 'spawnargs':iagent_args}) iagent_id = yield self.spawn_child(iapd) iaclient = InstrumentAgentClient(proc=self, target=iagent_id) yield iaclient.register_resource(inst_id) yield self.reply_ok(msg, "OK") @defer.inlineCallbacks def op_stop_instrument_agent(self, content, headers, msg): """ Service operation: Starts direct access mode. """ yield self.reply_err(msg, "Not yet implemented") @defer.inlineCallbacks def op_start_direct_access(self, content, headers, msg): """ Service operation: Starts direct access mode. """ yield self.reply_err(msg, "Not yet implemented") @defer.inlineCallbacks def op_stop_direct_access(self, content, headers, msg): """ Service operation: Stops direct access mode. """ yield self.reply_err(msg, "Not yet implemented") @defer.inlineCallbacks def get_agent_desc_for_instrument(self, instrument_id): logging.info("get_agent_desc_for_instrument() instrumentID="+str(instrument_id)) int_ref = ResourceReference(RegistryIdentity=instrument_id, RegistryBranch='master') agent_query = InstrumentAgentResourceInstance() agent_query.instrument_ref = int_ref if not agent_res: defer.returnValue(None) agent_pid = agent_res.proc_id logging.info("Agent process id for instrument id %s is: %s" % (instrument_id, agent_pid)) defer.returnValue(agent_pid) @defer.inlineCallbacks def get_agent_for_instrument(self, instrument_id): logging.info("get_agent_for_instrument() instrumentID="+str(instrument_id)) int_ref = ResourceReference(RegistryIdentity=instrument_id, RegistryBranch='master') agent_query = InstrumentAgentResourceInstance() agent_query.instrument_ref = int_ref # @todo Need to list the LC state here. WHY??? agent_query.lifecycle = LCStates.developed agents = yield self.arc.find_registered_agent_instance_from_description(agent_query, regex=False) logging.info("Found %s agent instances for instrument id %s" % (len(agents), instrument_id)) agent_res = None if len(agents) > 0: agent_res = agents[0] defer.returnValue(agent_res) @defer.inlineCallbacks def get_agent_pid_for_instrument(self, instrument_id): agent_res = yield self.get_agent_for_instrument(instrument_id) if not agent_res: defer.returnValue(None) agent_pid = agent_res.proc_id logging.info("Agent process id for instrument id %s is: %s" % (instrument_id, agent_pid)) defer.returnValue(agent_pid)