def test_sample(self):
result = yield self.driver_client.initialize('some arg')
dpsc = DataPubsubClient(self.sup)
topicname = 'SBE49 Topic'
topic = PubSubTopicResource.create(topicname,"")
# Use the service to create a queue and register the topic
topic = yield dpsc.define_topic(topic)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create(topicname,'')
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':'topic1'}
}
subscription = yield dpsc.define_subscription(subscription)
logging.info('Defined subscription: '+str(subscription))
params = {}
params['publish-to'] = topic.RegistryIdentity
yield self.driver_client.configure_driver(params)
cmd1 = [['ds', 'now']]
result = yield self.driver_client.execute(cmd1)
self.assertEqual(result['status'], 'OK')
yield pu.asleep(1)
result = yield self.driver_client.disconnect(['some arg'])
def test_create_topic(self):
#dpsc = DataPubsubClient(self.pubsubSuper)
dpsc = DataPubsubClient(self.sup)
# Create and Register a topic
"""
DHE: not sure the driver should be creating the topic; for right
now I'll have the test case do it.
"""
self.topic = PubSubTopicResource.create('SBE49 Topic',"oceans, oil spill")
self.topic = yield dpsc.define_topic(self.topic)
print 'TADA!'
def test_pubsub(self):
dpsc = DataPubsubClient(self.sup)
# Create and Register a topic
topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")
topic = yield dpsc.define_topic(topic)
logging.info('Defined Topic: '+str(topic))
#Create and register self.sup as a publisher
publisher = PublisherResource.create('Test Publisher', self.sup, topic, 'DataObject')
publisher = yield dpsc.define_publisher(publisher)
logging.info('Defined Publisher: '+str(publisher))
# === Create a Consumer and queues - this will become part of define_subscription.
#Create two test queues - don't use topics to test the consumer
# To be replaced when the subscription service is ready
queue1=dataobject.create_unique_identity()
queue_properties = {queue1:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
queue2=dataobject.create_unique_identity()
queue_properties = {queue2:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
pd1={'name':'example_consumer_1',
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',
'spawnargs':{'attach':topic.queue.name,\
'process parameters':{},\
'delivery queues':{'queues':[queue1,queue2]}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
# === End to be replaces with Define_Consumer
# Create and send a data message
data = {'Data':'in a dictionary'}
result = yield dpsc.publish(self.sup, topic.reference(), data)
if result:
logging.info('Published Message')
else:
logging.info('Failed to Published Message')
# Need to await the delivery of data messages into the (separate) consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(queue1),1)
self.assertEqual(sent.get(queue2),1)
self.assertEqual(received.get(topic.queue.name),1)
# === Create a Consumer - this will become part of define_subscription.
pd2={'name':'example_consumer_2',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':queue1,\
'process parameters':{},\
'delivery queues':{}}\
}
child2 = base_consumer.ConsumerDesc(**pd2)
child2_id = yield self.test_sup.spawn_child(child2)
# === End of what will become part of the subscription definition
# Send the simple message again
result = yield dpsc.publish(self.sup, topic.reference(), data)
# Need to await the delivery of data messages into the (separate) consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(queue1),2)
self.assertEqual(sent.get(queue2),2)
self.assertEqual(received.get(topic.queue.name),2)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(queue1),1)
def test_exampleconsumer(self):
'''
@Brief Example Consumer is a demonstration of a more complex data consumer.
It uses DAP data messages and provides qaqc and event results on two
seperate queues.
'''
dpsc = DataPubsubClient(self.sup)
#Create and register 3 topics!
topic_raw = PubSubTopicResource.create("topic_raw","oceans, oil spill, fun things to do")
topic_raw = yield dpsc.define_topic(topic_raw)
#Create and register self.sup as a publisher
publisher = PublisherResource.create('Test Publisher', self.sup, topic_raw, 'DataObject')
publisher = yield dpsc.define_publisher(publisher)
logging.info('Defined Publisher: '+str(publisher))
# === Create a Consumer and queues - this will become part of define_subscription.
#Create two test queues - don't use topics to test the consumer
# To be replaced when the subscription service is ready
evt_queue=dataobject.create_unique_identity()
queue_properties = {evt_queue:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
pr_queue=dataobject.create_unique_identity()
queue_properties = {pr_queue:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
pd1={'name':'example_consumer_1',
'module':'ion.services.dm.distribution.consumers.example_consumer',
'procclass':'ExampleConsumer',
'spawnargs':{'attach':topic_raw.queue.name,\
'Process Parameters':{},\
'delivery queues':\
{'event_queue':evt_queue,\
'processed_queue':pr_queue}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
pd2={'name':'example_consumer_2',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':evt_queue,\
'Process Parameters':{}}\
}
child2 = base_consumer.ConsumerDesc(**pd2)
child2_id = yield self.test_sup.spawn_child(child2)
pd3={'name':'example_consumer_3',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':pr_queue,\
'Process Parameters':{}}\
}
child3 = base_consumer.ConsumerDesc(**pd3)
child3_id = yield self.test_sup.spawn_child(child3)
# === End of stuff that will be replaced with Subscription method...
# Create an example data message
dmsg = dap_tools.simple_datamessage(\
{'DataSet Name':'Simple Data','variables':\
{'time':{'long_name':'Data and Time','units':'seconds'},\
'height':{'long_name':'person height','units':'meters'}}}, \
{'time':(101,102,103,104,105,106,107,108,109,110), \
'height':(5,2,4,5,-1,9,3,888,3,4)})
result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg)
if result:
logging.info('Published Message')
else:
logging.info('Failed to Published Message')
# Need to await the delivery of data messages into the consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(evt_queue),2)
self.assertEqual(sent.get(pr_queue),1)
self.assertEqual(received.get(topic_raw.queue.name),1)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(evt_queue),2)
msg_cnt = yield child3.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(pr_queue),1)
# Publish a second message with different data
dmsg = dap_tools.simple_datamessage(\
{'DataSet Name':'Simple Data','variables':\
{'time':{'long_name':'Data and Time','units':'seconds'},\
'height':{'long_name':'person height','units':'meters'}}}, \
{'time':(111,112,123,114,115,116,117,118,119,120), \
'height':(8,986,4,-2,-1,5,3,1,4,5)})
result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg)
# Need to await the delivery of data messages into the consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(evt_queue),5)
self.assertEqual(sent.get(pr_queue),2)
self.assertEqual(received.get(topic_raw.queue.name),2)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(evt_queue),5)
msg_cnt = yield child3.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(pr_queue),2)
class InstrumentManagementService(BaseService):
"""
Instrument management service interface.
This service provides overall coordination for instrument management within
an observatory context. In particular it coordinates the access to the
instrument and data product registries and the interaction with instrument
agents.
"""
# Declaration of service
declare = BaseService.service_declare(name='instrument_management',
version='0.1.0',
dependencies=[])
def slc_init(self):
self.irc = InstrumentRegistryClient(proc=self)
self.dprc = DataProductRegistryClient(proc=self)
self.arc = AgentRegistryClient(proc=self)
self.dpsc = DataPubsubClient(proc=self)
@defer.inlineCallbacks
def op_create_new_instrument(self, content, headers, msg):
"""
Service operation: Accepts a dictionary containing user inputs.
Updates the instrument registry.
"""
userInput = content['userInput']
newinstrument = InstrumentResource.create_new_resource()
if 'name' in userInput:
newinstrument.name = str(userInput['name'])
if 'description' in userInput:
newinstrument.description = str(userInput['description'])
if 'manufacturer' in userInput:
newinstrument.manufacturer = str(userInput['manufacturer'])
if 'model' in userInput:
newinstrument.model = str(userInput['model'])
if 'serial_num' in userInput:
newinstrument.serial_num = str(userInput['serial_num'])
if 'fw_version' in userInput:
newinstrument.fw_version = str(userInput['fw_version'])
instrument_res = yield self.irc.register_instrument_instance(newinstrument)
yield self.reply_ok(msg, instrument_res.encode())
@defer.inlineCallbacks
def op_create_new_data_product(self, content, headers, msg):
"""
Service operation: Accepts a dictionary containing user inputs.
Updates the data product registry. Also sets up an ingestion pipeline
for an instrument
"""
dataProductInput = content['dataProductInput']
newdp = DataProductResource.create_new_resource()
if 'instrumentID' in dataProductInput:
inst_id = str(dataProductInput['instrumentID'])
int_ref = ResourceReference(RegistryIdentity=inst_id, RegistryBranch='master')
newdp.instrument_ref = int_ref
if 'name' in dataProductInput:
newdp.name = str(dataProductInput['name'])
if 'description' in dataProductInput:
newdp.description = str(dataProductInput['description'])
if 'dataformat' in dataProductInput:
newdp.dataformat = str(dataProductInput['dataformat'])
# Step: Create a data stream
## Instantiate a pubsubclient
#self.dpsc = DataPubsubClient(proc=self)
#
## Create and Register a topic
#self.topic = PubSubTopicResource.create('SBE49 Topic',"oceans, oil spill")
#self.topic = yield self.dpsc.define_topic(self.topic)
#logging.debug('DHE: Defined Topic')
#
#self.publisher = PublisherResource.create('Test Publisher', self, self.topic, 'DataObject')
#self.publisher = yield self.dpsc.define_publisher(self.publisher)
res = yield self.dprc.register_data_product(newdp)
ref = res.reference(head=True)
yield self.reply_ok(msg, res.encode())
@defer.inlineCallbacks
def op_execute_command(self, content, headers, msg):
"""
Service operation: Execute a command on an instrument.
"""
# Step 1: Extract the arguments from the UI generated message content
commandInput = content['commandInput']
if 'instrumentID' in commandInput:
inst_id = str(commandInput['instrumentID'])
else:
raise ValueError("Input for instrumentID not present")
command = []
if 'command' in commandInput:
command_op = str(commandInput['command'])
else:
raise ValueError("Input for command not present")
command.append(command_op)
arg_idx = 0
while True:
argname = 'cmdArg'+str(arg_idx)
arg_idx += 1
if argname in commandInput:
command.append(str(commandInput[argname]))
else:
break
# Step 2: Find the agent id for the given instrument id
agent_pid = yield self.get_agent_pid_for_instrument(inst_id)
if not agent_pid:
yield self.reply_err(msg, "No agent found for instrument "+str(inst_id))
defer.returnValue(None)
# Step 3: Interact with the agent to execute the command
iaclient = InstrumentAgentClient(proc=self, target=agent_pid)
commandlist = [command,]
logging.info("Sending command to IA: "+str(commandlist))
cmd_result = yield iaclient.execute_instrument(commandlist)
yield self.reply_ok(msg, cmd_result)
@defer.inlineCallbacks
def op_get_instrument_state(self, content, headers, msg):
"""
Service operation: .
"""
# Step 1: Extract the arguments from the UI generated message content
commandInput = content['commandInput']
if 'instrumentID' in commandInput:
inst_id = str(commandInput['instrumentID'])
else:
raise ValueError("Input for instrumentID not present")
agent_pid = yield self.get_agent_pid_for_instrument(inst_id)
if not agent_pid:
raise StandardError("No agent found for instrument "+str(inst_id))
iaclient = InstrumentAgentClient(proc=self, target=agent_pid)
inst_cap = yield iaclient.get_capabilities()
if not inst_cap:
raise StandardError("No capabilities available for instrument "+str(inst_id))
ci_commands = inst_cap['ci_commands']
instrument_commands = inst_cap['instrument_commands']
instrument_parameters = inst_cap['instrument_parameters']
ci_parameters = inst_cap['ci_parameters']
values = yield iaclient.get_from_instrument(instrument_parameters)
resvalues = {}
if values:
resvalues = values
yield self.reply_ok(msg, resvalues)
@defer.inlineCallbacks
def op_start_instrument_agent(self, content, headers, msg):
"""
Service operation: Starts an instrument agent for a type of
instrument.
"""
if 'instrumentID' in content:
inst_id = str(content['instrumentID'])
else:
raise ValueError("Input for instrumentID not present")
if 'model' in content:
model = str(content['model'])
else:
raise ValueError("Input for model not present")
if model != 'SBE49':
raise ValueError("Only SBE49 supported!")
agent_pid = yield self.get_agent_pid_for_instrument(inst_id)
if agent_pid:
raise StandardError("Agent already started for instrument "+str(inst_id))
simulator = Simulator(inst_id)
simulator.start()
topicname = "Inst/RAW/"+inst_id
topic = PubSubTopicResource.create(topicname,"")
# Use the service to create a queue and register the topic
topic = yield self.dpsc.define_topic(topic)
iagent_args = {}
iagent_args['instrument-id'] = inst_id
driver_args = {}
driver_args['port'] = simulator.port
driver_args['publish-to'] = topic.RegistryIdentity
iagent_args['driver-args'] = driver_args
iapd = ProcessDesc(**{'name':'SBE49IA',
'module':'ion.agents.instrumentagents.SBE49_IA',
'class':'SBE49InstrumentAgent',
'spawnargs':iagent_args})
iagent_id = yield self.spawn_child(iapd)
iaclient = InstrumentAgentClient(proc=self, target=iagent_id)
yield iaclient.register_resource(inst_id)
yield self.reply_ok(msg, "OK")
@defer.inlineCallbacks
def op_stop_instrument_agent(self, content, headers, msg):
"""
Service operation: Starts direct access mode.
"""
yield self.reply_err(msg, "Not yet implemented")
@defer.inlineCallbacks
def op_start_direct_access(self, content, headers, msg):
"""
Service operation: Starts direct access mode.
"""
yield self.reply_err(msg, "Not yet implemented")
@defer.inlineCallbacks
def op_stop_direct_access(self, content, headers, msg):
"""
Service operation: Stops direct access mode.
"""
yield self.reply_err(msg, "Not yet implemented")
@defer.inlineCallbacks
def get_agent_desc_for_instrument(self, instrument_id):
logging.info("get_agent_desc_for_instrument() instrumentID="+str(instrument_id))
int_ref = ResourceReference(RegistryIdentity=instrument_id, RegistryBranch='master')
agent_query = InstrumentAgentResourceInstance()
agent_query.instrument_ref = int_ref
if not agent_res:
defer.returnValue(None)
agent_pid = agent_res.proc_id
logging.info("Agent process id for instrument id %s is: %s" % (instrument_id, agent_pid))
defer.returnValue(agent_pid)
@defer.inlineCallbacks
def get_agent_for_instrument(self, instrument_id):
logging.info("get_agent_for_instrument() instrumentID="+str(instrument_id))
int_ref = ResourceReference(RegistryIdentity=instrument_id, RegistryBranch='master')
agent_query = InstrumentAgentResourceInstance()
agent_query.instrument_ref = int_ref
# @todo Need to list the LC state here. WHY???
agent_query.lifecycle = LCStates.developed
agents = yield self.arc.find_registered_agent_instance_from_description(agent_query, regex=False)
logging.info("Found %s agent instances for instrument id %s" % (len(agents), instrument_id))
agent_res = None
if len(agents) > 0:
agent_res = agents[0]
defer.returnValue(agent_res)
@defer.inlineCallbacks
def get_agent_pid_for_instrument(self, instrument_id):
agent_res = yield self.get_agent_for_instrument(instrument_id)
if not agent_res:
defer.returnValue(None)
agent_pid = agent_res.proc_id
logging.info("Agent process id for instrument id %s is: %s" % (instrument_id, agent_pid))
defer.returnValue(agent_pid)
