def test_sample(self):
result = yield self.driver_client.initialize('some arg')
dpsc = DataPubsubClient(self.sup)
topicname = 'SBE49 Topic'
topic = PubSubTopicResource.create(topicname,"")
# Use the service to create a queue and register the topic
topic = yield dpsc.define_topic(topic)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create(topicname,'')
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':'topic1'}
}
subscription = yield dpsc.define_subscription(subscription)
logging.info('Defined subscription: '+str(subscription))
params = {}
params['publish-to'] = topic.RegistryIdentity
yield self.driver_client.configure_driver(params)
cmd1 = [['ds', 'now']]
result = yield self.driver_client.execute(cmd1)
self.assertEqual(result['status'], 'OK')
yield pu.asleep(1)
result = yield self.driver_client.disconnect(['some arg'])
def test_sample(self):
result = yield self.driver_client.initialize('some arg')
dpsc = DataPubsubClient(self.sup)
topicname = 'SBE49 Topic'
topic = PubSubTopicResource.create(topicname, "")
# Use the service to create a queue and register the topic
topic = yield dpsc.define_topic(topic)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create(topicname, '')
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':'topic1'}
}
subscription = yield dpsc.define_subscription(subscription)
logging.info('Defined subscription: ' + str(subscription))
params = {}
params['publish-to'] = topic.RegistryIdentity
yield self.driver_client.configure_driver(params)
cmd1 = [['ds', 'now']]
result = yield self.driver_client.execute(cmd1)
self.assertEqual(result['status'], 'OK')
yield pu.asleep(1)
result = yield self.driver_client.disconnect(['some arg'])
def test_create_consumer_args1(self):
'''
@Brief Test business logic to create consumer args from a workflow
Make sure teh basics of the workflow processing work to create consumer
args.
'''
# Create and Register a topic
topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")
# Use the service to create a queue and register the topic
topic = yield self.pubsub.create_and_register_topic(topic)
# A condition to search for the topic
t_search = PubSubTopicResource.create('Davids Topic','')
subscription = SubscriptionResource()
subscription.topic1 = t_search
subscription.workflow = {'consumer1':{'module':'path.to.module','consumerclass':'<ConsumerClassName>',\
'attach':'topic1',\
'process parameters':{'param1':'my parameter'}}}
subscription = yield self.pubsub.create_consumer_args(subscription)
logging.debug('Defined subscription consumers: '+str(subscription.consumer_args))
self.assertEqual({'consumer1':
{'procclass': '<ConsumerClassName>',
'name': 'consumer1',
'module': 'path.to.module',
'spawnargs': {
'attach': [topic.queue.name],
'delivery interval': None,
'delivery queues': {},
'process parameters': {'param1': 'my parameter'}}}}, subscription.consumer_args)
def test_create_consumer_args3(self):
'''
@Brief Test business logic to create consumer args from a workflow
In this test two consumers attach to the same queue created by the workflow
'''
# Create and Register a topic
topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")
# Use the service to create a queue and register the topic
topic = yield self.pubsub.create_and_register_topic(topic)
# A condition to search for the topic
t_search = PubSubTopicResource.create('Davids Topic','')
subscription = SubscriptionResource()
subscription.topic1 = t_search
subscription.workflow = {
'consumer1':{'module':'path.to.module','consumerclass':'<ConsumerClassName>',\
'attach':'topic1',\
'process parameters':{'param1':'my parameter1'}},
'consumer2':{'module':'path.to.module2','consumerclass':'<ConsumerClassName2>',\
'attach':[['consumer1','deliver_to']],\
'Process Parameters':{'param2':'my parameter2'}},
'consumer3':{'module':'path.to.module3','consumerclass':'<ConsumerClassName3>',\
'attach':[['consumer1','deliver_to'],'topic1']}
}
subscription = yield self.pubsub.create_consumer_args(subscription)
logging.debug('Defined subscription consumers: '+str(subscription.consumer_args))
consume2_attach = subscription.consumer_args['consumer2']['spawnargs']['attach']
self.assertEqual({'module': 'path.to.module',
'name': 'consumer1',
'procclass': '<ConsumerClassName>',
'spawnargs': {'attach': [topic.queue.name],
'delivery interval': None,
'delivery queues': {'deliver_to': consume2_attach[0]},
'process parameters': {'param1': 'my parameter1'}}},
subscription.consumer_args['consumer1'])
self.assertEqual({'procclass': '<ConsumerClassName3>',
'name': 'consumer3',
'module': 'path.to.module3',
'spawnargs': {
'attach': [consume2_attach[0], topic.queue.name],
'delivery interval': None,
'delivery queues': {},
'process parameters': {}}},
subscription.consumer_args['consumer3'])
def test_create_subscription2(self):
'''
@Brief Create a subscription!
'''
# Create and Register a topic
topic1 = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")
# Use the service to create a queue and register the topic
topic1 = yield self.pubsub.create_and_register_topic(topic1)
topic2 = PubSubTopicResource.create('Johns Topic',"oceans, mbari, working really hard")
# Use the service to create a queue and register the topic
topic2 = yield self.pubsub.create_and_register_topic(topic2)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create('Davids Topic','')
subscription.topic2 = PubSubTopicResource.create('','oceans')
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'consumerclass':'ForwardingConsumer',\
'attach':'topic1'},
'consumer2':
{'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'consumerclass':'ForwardingConsumer',\
'attach':[['consumer1','queues']]}
}
subscription = yield self.pubsub.create_subscription(subscription)
logging.info('Defined subscription: '+str(subscription))
msg=DataMessageObject()
self.sup.send(topic1.queue.name,'data',msg.encode())
# Wait for message to be received
yield pu.asleep(1)
child1_id = self.pubsub.get_child_id('consumer1')
dc1 = self._get_procinstance(child1_id)
rec = dc1.receive_cnt[topic1.queue.name]
self.assertEqual(rec,1)
child2_id = self.pubsub.get_child_id('consumer2')
dc2 = self._get_procinstance(child2_id)
q = subscription.consumer_args['consumer2']['spawnargs']['attach']
rec = dc2.receive_cnt[q[0]]
self.assertEqual(rec,1)
def create_producers(proc,n=1):
dpsc = pubsub_service.DataPubsubClient(proc=proc)
for i in range(n):
tname = 'topic name '+str(i)
ka = 'keyword a'
kb = 'keyword b'
if (i/2)*2 == i:
keyword = ka
else:
keyword = kb
topic = PubSubTopicResource.create(tname,keyword)
topic = yield dpsc.define_topic(topic)
dspname = 'data_stream_producer_'+str(i)
interval = random.randint(1,10)
dsp={'name':dspname,
'module':'ion.services.dm.util.data_stream_producer',
'procclass':'DataStreamProducer',
'spawnargs':{'delivery queue':topic.queue.name,
'delivery interval':interval}}
child = ProcessDesc(**dsp)
child_id = yield proc.spawn_child(child)
def create_producers(proc, n=1):
dpsc = pubsub_service.DataPubsubClient(proc=proc)
for i in range(n):
tname = 'topic name ' + str(i)
ka = 'keyword a'
kb = 'keyword b'
if (i / 2) * 2 == i:
keyword = ka
else:
keyword = kb
topic = PubSubTopicResource.create(tname, keyword)
topic = yield dpsc.define_topic(topic)
dspname = 'data_stream_producer_' + str(i)
interval = random.randint(1, 10)
dsp = {
'name': dspname,
'module': 'ion.services.dm.util.data_stream_producer',
'procclass': 'DataStreamProducer',
'spawnargs': {
'delivery queue': topic.queue.name,
'delivery interval': interval
}
}
child = ProcessDesc(**dsp)
child_id = yield proc.spawn_child(child)
def test_create_topic(self):
# Create and Register a topic
topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")
# Make sure the topic starts out with blank stuff here...
self.assertEqual(topic.RegistryBranch,'master')
self.assertEqual(topic.RegistryIdentity,'')
self.assertEqual(topic.queue.type,'')
# Use the service to create a queue and register the topic
topic = yield self.pubsub.create_and_register_topic(topic)
# Make sure the queue properties were set.
self.assertEqual(topic.queue.type,'fanout')
#Spawn an baseconsumer and make sure a message is received on the new queue
pd1={'name':'consumer_number_1',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':[topic.queue.name]}}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
msg=DataMessageObject()
self.sup.send(topic.queue.name,'data',msg.encode())
yield pu.asleep(1)
dc1 = self._get_procinstance(child1_id)
#print 'DC1',dc1.receive_cnt
rec = dc1.receive_cnt[topic.queue.name]
self.assertEqual(rec,1)
def op_start_instrument_agent(self, content, headers, msg):
"""
Service operation: Starts an instrument agent for a type of
instrument.
"""
if 'instrumentID' in content:
inst_id = str(content['instrumentID'])
else:
raise ValueError("Input for instrumentID not present")
if 'model' in content:
model = str(content['model'])
else:
raise ValueError("Input for model not present")
if model != 'SBE49':
raise ValueError("Only SBE49 supported!")
agent_pid = yield self.get_agent_pid_for_instrument(inst_id)
if agent_pid:
raise StandardError("Agent already started for instrument " +
str(inst_id))
simulator = Simulator(inst_id)
simulator.start()
topicname = "Inst/RAW/" + inst_id
topic = PubSubTopicResource.create(topicname, "")
# Use the service to create a queue and register the topic
topic = yield self.dpsc.define_topic(topic)
iagent_args = {}
iagent_args['instrument-id'] = inst_id
driver_args = {}
driver_args['port'] = simulator.port
driver_args['publish-to'] = topic.RegistryIdentity
iagent_args['driver-args'] = driver_args
iapd = ProcessDesc(
**{
'name': 'SBE49IA',
'module': 'ion.agents.instrumentagents.SBE49_IA',
'class': 'SBE49InstrumentAgent',
'spawnargs': iagent_args
})
iagent_id = yield self.spawn_child(iapd)
iaclient = InstrumentAgentClient(proc=self, target=iagent_id)
yield iaclient.register_resource(inst_id)
yield self.reply_ok(msg, "OK")
def start():
"""
Main function of bootstrap. Starts DM pubsub...
"""
logging.info("ION DM PubSub bootstrapping now...")
startsvcs = []
#startsvcs.extend(dm_services)
sup = yield bootstrap.bootstrap(ion_messaging, startsvcs)
print 'STARTSVCS',startsvcs
print 'ION_MESSAGING',ion_messaging
print 'CONT_ARGS',ioninit.cont_args
#yield create_producer(sup)
dpsc = pubsub_service.DataPubsubClient(proc=sup)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create('Inst/RAW','')
# Use the example consumer to create events... graph the number of events
'''
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':'topic1'}
}
'''
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.instrument_timeseries_consumer',
'consumerclass':'InstrumentTimeseriesConsumer',\
'attach':'topic1'},
'consumer3':
{'module':'ion.services.dm.presentation.web_viz_consumer',
'consumerclass':'WebVizConsumer',\
'attach':[['consumer1','queue']],
'process parameters':{'port':8180}
}
}
subscription = yield dpsc.define_subscription(subscription)
linfo = '\n================================================\n'
linfo+= 'Open your web browser and look at: http://127.0.0.1:8180/ \n'
linfo+= '================================================\n'
logging.info(linfo)
def start():
"""
Main function of bootstrap. Starts DM pubsub...
"""
logging.info("ION DM PubSub bootstrapping now...")
startsvcs = []
#startsvcs.extend(dm_services)
sup = yield bootstrap.bootstrap(ion_messaging, startsvcs)
print 'STARTSVCS', startsvcs
print 'ION_MESSAGING', ion_messaging
print 'CONT_ARGS', ioninit.cont_args
#yield create_producer(sup)
dpsc = pubsub_service.DataPubsubClient(proc=sup)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create('Inst/RAW', '')
# Use the example consumer to create events... graph the number of events
'''
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':'topic1'}
}
'''
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.instrument_timeseries_consumer',
'consumerclass':'InstrumentTimeseriesConsumer',\
'attach':'topic1'},
'consumer3':
{'module':'ion.services.dm.presentation.web_viz_consumer',
'consumerclass':'WebVizConsumer',\
'attach':[['consumer1','queue']],
'process parameters':{'port':8180}
}
}
subscription = yield dpsc.define_subscription(subscription)
linfo = '\n================================================\n'
linfo += 'Open your web browser and look at: http://127.0.0.1:8180/ \n'
linfo += '================================================\n'
logging.info(linfo)
def test_create_topic(self):
#dpsc = DataPubsubClient(self.pubsubSuper)
dpsc = DataPubsubClient(self.sup)
# Create and Register a topic
"""
DHE: not sure the driver should be creating the topic; for right
now I'll have the test case do it.
"""
self.topic = PubSubTopicResource.create('SBE49 Topic',
"oceans, oil spill")
self.topic = yield dpsc.define_topic(self.topic)
print 'TADA!'
def test_create_topic(self):
#dpsc = DataPubsubClient(self.pubsubSuper)
dpsc = DataPubsubClient(self.sup)
# Create and Register a topic
"""
DHE: not sure the driver should be creating the topic; for right
now I'll have the test case do it.
"""
self.topic = PubSubTopicResource.create('SBE49 Topic',"oceans, oil spill")
self.topic = yield dpsc.define_topic(self.topic)
print 'TADA!'
def test_create_subscription1(self):
'''
@Brief Create a subscription!
'''
# Create and Register a topic
topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")
# Use the service to create a queue and register the topic
topic = yield self.pubsub.create_and_register_topic(topic)
# A condition to search for the topic
t_search = PubSubTopicResource.create('Davids Topic','')
subscription = SubscriptionResource()
subscription.topic1 = t_search
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'consumerclass':'ForwardingConsumer',\
'attach':'topic1'}}
subscription = yield self.pubsub.create_subscription(subscription)
logging.info('Defined subscription: '+str(subscription))
msg=DataMessageObject()
self.sup.send(topic.queue.name,'data',msg.encode())
# Wait for message to be received
yield pu.asleep(1)
child_id = self.pubsub.get_child_id('consumer1')
dc1 = self._get_procinstance(child_id)
rec = dc1.receive_cnt[topic.queue.name]
self.assertEqual(rec,1)
def op_start_instrument_agent(self, content, headers, msg):
"""
Service operation: Starts an instrument agent for a type of
instrument.
"""
if 'instrumentID' in content:
inst_id = str(content['instrumentID'])
else:
raise ValueError("Input for instrumentID not present")
if 'model' in content:
model = str(content['model'])
else:
raise ValueError("Input for model not present")
if model != 'SBE49':
raise ValueError("Only SBE49 supported!")
agent_pid = yield self.get_agent_pid_for_instrument(inst_id)
if agent_pid:
raise StandardError("Agent already started for instrument "+str(inst_id))
simulator = Simulator(inst_id)
simulator.start()
topicname = "Inst/RAW/"+inst_id
topic = PubSubTopicResource.create(topicname,"")
# Use the service to create a queue and register the topic
topic = yield self.dpsc.define_topic(topic)
iagent_args = {}
iagent_args['instrument-id'] = inst_id
driver_args = {}
driver_args['port'] = simulator.port
driver_args['publish-to'] = topic.RegistryIdentity
iagent_args['driver-args'] = driver_args
iapd = ProcessDesc(**{'name':'SBE49IA',
'module':'ion.agents.instrumentagents.SBE49_IA',
'class':'SBE49InstrumentAgent',
'spawnargs':iagent_args})
iagent_id = yield self.spawn_child(iapd)
iaclient = InstrumentAgentClient(proc=self, target=iagent_id)
yield iaclient.register_resource(inst_id)
yield self.reply_ok(msg, "OK")
def create_producer(proc):
dpsc = pubsub_service.DataPubsubClient(proc=proc)
topic = PubSubTopicResource.create('time series','dap grid, timeseries, data')
topic = yield dpsc.define_topic(topic)
dspname = 'data_timeseries_stream_producer'
dsp={'name':dspname,
'module':'ion.services.dm.util.dap_grid_timeseries_producer',
'procclass':'DapGridDataStreamProducer',
'spawnargs':{'delivery queue':topic.queue.name,
'delivery interval':5}
}
child = ProcessDesc(**dsp)
child_id = yield proc.spawn_child(child)
def create_producer(proc):
dpsc = pubsub_service.DataPubsubClient(proc=proc)
topic = PubSubTopicResource.create('time series',
'dap grid, timeseries, data')
topic = yield dpsc.define_topic(topic)
dspname = 'data_timeseries_stream_producer'
dsp = {
'name': dspname,
'module': 'ion.services.dm.util.dap_grid_timeseries_producer',
'procclass': 'DapGridDataStreamProducer',
'spawnargs': {
'delivery queue': topic.queue.name,
'delivery interval': 5
}
}
child = ProcessDesc(**dsp)
child_id = yield proc.spawn_child(child)
def test_execute(self):
"""
Test the execute command to the Instrument Driver
"""
result = yield self.driver_client.initialize('some arg')
dpsc = DataPubsubClient(self.sup)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create('SBE49 Topic','')
#subscription.topic2 = PubSubTopicResource.create('','oceans')
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':'topic1'}
}
subscription = yield dpsc.define_subscription(subscription)
logging.info('Defined subscription: '+str(subscription))
cmd1 = [['ds', 'now']]
#cmd1 = [['start', 'now']]
#cmd2 = [['stop', 'now']]
#cmd2 = [['pumpoff', '3600', '1']]
result = yield self.driver_client.execute(cmd1)
self.assertEqual(result['status'], 'OK')
# DHE: wait a while...
yield pu.asleep(1)
#result = yield self.driver_client.execute(cmd2)
#self.assertEqual(result['status'], 'OK')
# DHE: disconnecting; a connect would probably be good.
result = yield self.driver_client.disconnect(['some arg'])
def test_execute(self):
"""
Test the execute command to the Instrument Driver
"""
result = yield self.driver_client.initialize('some arg')
dpsc = DataPubsubClient(self.sup)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create('SBE49 Topic', '')
#subscription.topic2 = PubSubTopicResource.create('','oceans')
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':'topic1'}
}
subscription = yield dpsc.define_subscription(subscription)
logging.info('Defined subscription: ' + str(subscription))
cmd1 = [['ds', 'now']]
#cmd1 = [['start', 'now']]
#cmd2 = [['stop', 'now']]
#cmd2 = [['pumpoff', '3600', '1']]
result = yield self.driver_client.execute(cmd1)
self.assertEqual(result['status'], 'OK')
# DHE: wait a while...
yield pu.asleep(1)
#result = yield self.driver_client.execute(cmd2)
#self.assertEqual(result['status'], 'OK')
# DHE: disconnecting; a connect would probably be good.
result = yield self.driver_client.disconnect(['some arg'])
def test_pubsub(self):
dpsc = DataPubsubClient(self.sup)
# Create and Register a topic
topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")
topic = yield dpsc.define_topic(topic)
logging.info('Defined Topic: '+str(topic))
#Create and register self.sup as a publisher
publisher = PublisherResource.create('Test Publisher', self.sup, topic, 'DataObject')
publisher = yield dpsc.define_publisher(publisher)
logging.info('Defined Publisher: '+str(publisher))
# === Create a Consumer and queues - this will become part of define_subscription.
#Create two test queues - don't use topics to test the consumer
# To be replaced when the subscription service is ready
queue1=dataobject.create_unique_identity()
queue_properties = {queue1:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
queue2=dataobject.create_unique_identity()
queue_properties = {queue2:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
pd1={'name':'example_consumer_1',
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',
'spawnargs':{'attach':topic.queue.name,\
'process parameters':{},\
'delivery queues':{'queues':[queue1,queue2]}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
# === End to be replaces with Define_Consumer
# Create and send a data message
data = {'Data':'in a dictionary'}
result = yield dpsc.publish(self.sup, topic.reference(), data)
if result:
logging.info('Published Message')
else:
logging.info('Failed to Published Message')
# Need to await the delivery of data messages into the (separate) consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(queue1),1)
self.assertEqual(sent.get(queue2),1)
self.assertEqual(received.get(topic.queue.name),1)
# === Create a Consumer - this will become part of define_subscription.
pd2={'name':'example_consumer_2',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':queue1,\
'process parameters':{},\
'delivery queues':{}}\
}
child2 = base_consumer.ConsumerDesc(**pd2)
child2_id = yield self.test_sup.spawn_child(child2)
# === End of what will become part of the subscription definition
# Send the simple message again
result = yield dpsc.publish(self.sup, topic.reference(), data)
# Need to await the delivery of data messages into the (separate) consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(queue1),2)
self.assertEqual(sent.get(queue2),2)
self.assertEqual(received.get(topic.queue.name),2)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(queue1),1)
def op_create_ingestion_datastream(self, content, headers, msg):
"""Service operation: declare new named datastream for ingestion
0) Decode content to Igestion Stream Resource
1) create inbound topic
2) create ingested topic
3) Register new data
3) start preservation of ingested
4) start preservation of inbound
5) start ingestion workflow
Register progress and set lcstate along the way
return the ingestiondatastream resource
"""
logging.debug(self.__class__.__name__ +', op_'+ headers['op'] +' Received: ' + str(headers))
isr = dataobject.DataObject.decode(content)
logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', Ingestion Stream: \n' + str(publisher))
# Register the intended feed...
isr = yield self.reg.define_ingestion_stream(isr)
inbnd = irs.name + '.inbound'
topic = PubSubTopicResource.create(name=inbnd,keywords='input') # Place holder - what to put, NOT RAW!
topic = yield self.pubsub.define_topic(topic)
isr.input_topic = topic.reference(heat=True)
ingested = irs.name + '.ingested'
topic = PubSubTopicResource.create(name=ingested,keywords='ingested')
topic = yield self.pubsub.define_topic(topic)
isr.ingested_topic = topic.reference(heat=True)
# Register the feed topics...
isr = yield self.reg.define_ingestion_stream(isr)
#Create a DM Data Resource for this new feed.
dmdr = DMDataResource.create_new_resource()
dmdr.input_topic = isr.input_topic
dmdr.ingested_topic = isr.ingested_topic
# Register the data
dmdr = yield self.datareg.define_data(dmdr)
#Call the preservation service for the input stream
arc = ArchiveResource.create_new_resource()
arc.dmdataresource = dmdr.reference(head=true)
arc.topic = isr.input_topic
yield self.preserv.create_archive(arc)
yield self.preserv.activate_persister
# Register the data
dmdr.input_archive = arc.reference(head=True)
dmdr = yield self.datareg.define_data(dmdr)
#Call the preservation service for the ingested stream
arc = ArchiveResource.create_new_resource()
arc.dmdataresource = dmdr.reference(head=true)
arc.topic = isr.ingested_topic
yield self.preserv.create_archive(arc)
yield self.preserv.activate_persister
# Register the data
dmdr.ingested_archive = arc.reference(head=True)
dmdr = yield self.datareg.define_data(dmdr)
# Register the persisters
isr.persisting_inout = True
isr.persisting_ingested = True
isr = yield self.reg.define_ingestion_stream(isr)
#activate ingestion
isr = yield self._activate_ingestion(isr, dmdr)
isr = yield self.reg.define_ingestion_stream(isr)
yield self.reply_ok(isr.encode())
def start():
"""
Main function of bootstrap. Starts DM pubsub...
"""
logging.info("ION DM PubSub bootstrapping now...")
startsvcs = []
startsvcs.extend(dm_services)
sup = yield bootstrap.bootstrap(ion_messaging, startsvcs)
print 'STARTSVCS', startsvcs
print 'ION_MESSAGING', ion_messaging
print 'CONT_ARGS', ioninit.cont_args
nproducers = int(ioninit.cont_args.get('nproducers', 5))
print 'NPRODUCERS', nproducers
yield create_producers(sup, nproducers)
dpsc = pubsub_service.DataPubsubClient(proc=sup)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create('topic', '')
"""
# Use the example consumer to create events... log the events
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.example_consumer',
'consumerclass':'ExampleConsumer',\
'attach':'topic1'},
'consumer2':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':[['consumer1','event_queue']]}
}
# Log all the messages created
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':'topic1'}
}
"""
# Use the example consumer to create events... graph the number of events
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.example_consumer',
'consumerclass':'ExampleConsumer',\
'attach':'topic1'},
'consumer2':
{'module':'ion.services.dm.distribution.consumers.message_count_consumer',
'consumerclass':'MessageCountConsumer',\
'attach':[['consumer1','event_queue']],\
'delivery interval':5,
'process parameters':{'max_points':5}
},
'consumer3':
{'module':'ion.services.dm.presentation.web_viz_consumer',
'consumerclass':'WebVizConsumer',\
'attach':[['consumer2','queue']],
'process parameters':{'port':8180}
}
}
subscription = yield dpsc.define_subscription(subscription)
linfo = '\n================================================\n'
linfo += 'Open your web browser and look at: http://127.0.0.1:8180/ \n'
linfo += '================================================\n'
logging.info(linfo)
def test_exampleconsumer(self):
'''
@Brief Example Consumer is a demonstration of a more complex data consumer.
It uses DAP data messages and provides qaqc and event results on two
seperate queues.
'''
dpsc = DataPubsubClient(self.sup)
#Create and register 3 topics!
topic_raw = PubSubTopicResource.create("topic_raw","oceans, oil spill, fun things to do")
topic_raw = yield dpsc.define_topic(topic_raw)
#Create and register self.sup as a publisher
publisher = PublisherResource.create('Test Publisher', self.sup, topic_raw, 'DataObject')
publisher = yield dpsc.define_publisher(publisher)
logging.info('Defined Publisher: '+str(publisher))
# === Create a Consumer and queues - this will become part of define_subscription.
#Create two test queues - don't use topics to test the consumer
# To be replaced when the subscription service is ready
evt_queue=dataobject.create_unique_identity()
queue_properties = {evt_queue:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
pr_queue=dataobject.create_unique_identity()
queue_properties = {pr_queue:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
pd1={'name':'example_consumer_1',
'module':'ion.services.dm.distribution.consumers.example_consumer',
'procclass':'ExampleConsumer',
'spawnargs':{'attach':topic_raw.queue.name,\
'Process Parameters':{},\
'delivery queues':\
{'event_queue':evt_queue,\
'processed_queue':pr_queue}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
pd2={'name':'example_consumer_2',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':evt_queue,\
'Process Parameters':{}}\
}
child2 = base_consumer.ConsumerDesc(**pd2)
child2_id = yield self.test_sup.spawn_child(child2)
pd3={'name':'example_consumer_3',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':pr_queue,\
'Process Parameters':{}}\
}
child3 = base_consumer.ConsumerDesc(**pd3)
child3_id = yield self.test_sup.spawn_child(child3)
# === End of stuff that will be replaced with Subscription method...
# Create an example data message
dmsg = dap_tools.simple_datamessage(\
{'DataSet Name':'Simple Data','variables':\
{'time':{'long_name':'Data and Time','units':'seconds'},\
'height':{'long_name':'person height','units':'meters'}}}, \
{'time':(101,102,103,104,105,106,107,108,109,110), \
'height':(5,2,4,5,-1,9,3,888,3,4)})
result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg)
if result:
logging.info('Published Message')
else:
logging.info('Failed to Published Message')
# Need to await the delivery of data messages into the consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(evt_queue),2)
self.assertEqual(sent.get(pr_queue),1)
self.assertEqual(received.get(topic_raw.queue.name),1)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(evt_queue),2)
msg_cnt = yield child3.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(pr_queue),1)
# Publish a second message with different data
dmsg = dap_tools.simple_datamessage(\
{'DataSet Name':'Simple Data','variables':\
{'time':{'long_name':'Data and Time','units':'seconds'},\
'height':{'long_name':'person height','units':'meters'}}}, \
{'time':(111,112,123,114,115,116,117,118,119,120), \
'height':(8,986,4,-2,-1,5,3,1,4,5)})
result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg)
# Need to await the delivery of data messages into the consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(evt_queue),5)
self.assertEqual(sent.get(pr_queue),2)
self.assertEqual(received.get(topic_raw.queue.name),2)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(evt_queue),5)
msg_cnt = yield child3.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(pr_queue),2)
def start():
"""
Main function of bootstrap. Starts DM pubsub...
"""
logging.info("ION DM PubSub bootstrapping now...")
startsvcs = []
startsvcs.extend(dm_services)
sup = yield bootstrap.bootstrap(ion_messaging, startsvcs)
print 'STARTSVCS',startsvcs
print 'ION_MESSAGING',ion_messaging
print 'CONT_ARGS',ioninit.cont_args
nproducers = int(ioninit.cont_args.get('nproducers',5))
print 'NPRODUCERS',nproducers
yield create_producers(sup, nproducers)
dpsc = pubsub_service.DataPubsubClient(proc=sup)
subscription = SubscriptionResource()
subscription.topic1 = PubSubTopicResource.create('topic','')
"""
# Use the example consumer to create events... log the events
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.example_consumer',
'consumerclass':'ExampleConsumer',\
'attach':'topic1'},
'consumer2':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':[['consumer1','event_queue']]}
}
# Log all the messages created
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.logging_consumer',
'consumerclass':'LoggingConsumer',\
'attach':'topic1'}
}
"""
# Use the example consumer to create events... graph the number of events
subscription.workflow = {
'consumer1':
{'module':'ion.services.dm.distribution.consumers.example_consumer',
'consumerclass':'ExampleConsumer',\
'attach':'topic1'},
'consumer2':
{'module':'ion.services.dm.distribution.consumers.message_count_consumer',
'consumerclass':'MessageCountConsumer',\
'attach':[['consumer1','event_queue']],\
'delivery interval':5,
'process parameters':{'max_points':5}
},
'consumer3':
{'module':'ion.services.dm.presentation.web_viz_consumer',
'consumerclass':'WebVizConsumer',\
'attach':[['consumer2','queue']],
'process parameters':{'port':8180}
}
}
subscription = yield dpsc.define_subscription(subscription)
linfo = '\n================================================\n'
linfo+= 'Open your web browser and look at: http://127.0.0.1:8180/ \n'
linfo+= '================================================\n'
logging.info(linfo)
def op_create_ingestion_datastream(self, content, headers, msg):
"""Service operation: declare new named datastream for ingestion
0) Decode content to Igestion Stream Resource
1) create inbound topic
2) create ingested topic
3) Register new data
3) start preservation of ingested
4) start preservation of inbound
5) start ingestion workflow
Register progress and set lcstate along the way
return the ingestiondatastream resource
"""
logging.debug(self.__class__.__name__ + ", op_" + headers["op"] + " Received: " + str(headers))
isr = dataobject.DataObject.decode(content)
logging.info(
self.__class__.__name__ + " recieved: op_" + headers["op"] + ", Ingestion Stream: \n" + str(publisher)
)
# Register the intended feed...
isr = yield self.reg.define_ingestion_stream(isr)
inbnd = irs.name + ".inbound"
topic = PubSubTopicResource.create(name=inbnd, keywords="input") # Place holder - what to put, NOT RAW!
topic = yield self.pubsub.define_topic(topic)
isr.input_topic = topic.reference(heat=True)
ingested = irs.name + ".ingested"
topic = PubSubTopicResource.create(name=ingested, keywords="ingested")
topic = yield self.pubsub.define_topic(topic)
isr.ingested_topic = topic.reference(heat=True)
# Register the feed topics...
isr = yield self.reg.define_ingestion_stream(isr)
# Create a DM Data Resource for this new feed.
dmdr = DMDataResource.create_new_resource()
dmdr.input_topic = isr.input_topic
dmdr.ingested_topic = isr.ingested_topic
# Register the data
dmdr = yield self.datareg.define_data(dmdr)
# Call the preservation service for the input stream
arc = ArchiveResource.create_new_resource()
arc.dmdataresource = dmdr.reference(head=true)
arc.topic = isr.input_topic
yield self.preserv.create_archive(arc)
yield self.preserv.activate_persister
# Register the data
dmdr.input_archive = arc.reference(head=True)
dmdr = yield self.datareg.define_data(dmdr)
# Call the preservation service for the ingested stream
arc = ArchiveResource.create_new_resource()
arc.dmdataresource = dmdr.reference(head=true)
arc.topic = isr.ingested_topic
yield self.preserv.create_archive(arc)
yield self.preserv.activate_persister
# Register the data
dmdr.ingested_archive = arc.reference(head=True)
dmdr = yield self.datareg.define_data(dmdr)
# Register the persisters
isr.persisting_inout = True
isr.persisting_ingested = True
isr = yield self.reg.define_ingestion_stream(isr)
# activate ingestion
isr = yield self._activate_ingestion(isr, dmdr)
isr = yield self.reg.define_ingestion_stream(isr)
yield self.reply_ok(isr.encode())
