def test_send_chain(self):
pd1={'name':'consumer_number_1',
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',
'spawnargs':{'attach':self.queue1,\
'process parameters':{},\
'delivery queues':{'queues':[self.queue2]}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
dmsg = DataMessageObject()
dmsg.notifcation = 'Junk'
dmsg.timestamp = pu.currenttime()
dmsg = dmsg.encode()
yield self.test_sup.send(self.queue1, 'data', dmsg)
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received', {})
sent = msg_cnt.get('sent', {})
self.assertEqual(sent.get(self.queue2), 1)
self.assertEqual(received.get(self.queue1), 1)
#Spawn another process to listen to queue 2
pd2={'name':'consumer_number_2', \
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',\
'spawnargs':{'attach':self.queue2}}
child2 = base_consumer.ConsumerDesc(**pd2)
child2_id = yield self.test_sup.spawn_child(child2)
# Tell the first consumer to pass results to the second!
#res = yield child1.set_params({'queues':[self.queue2]})
yield self.test_sup.send(self.queue1, 'data', dmsg)
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received', {})
sent = msg_cnt.get('sent', {})
self.assertEqual(sent.get(self.queue2), 2)
self.assertEqual(received.get(self.queue1), 2)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received', {})
sent = msg_cnt.get('sent')
self.assertEqual(sent, {})
self.assertEqual(received.get(self.queue2), 1)
yield child1.shutdown()
yield child2.shutdown()
def test_params(self):
pd1={'name':'consumer_number_1', \
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
# Send a dictionary
params = {'Junk': 'Trunk'}
res = yield child1.set_process_parameters(params)
self.assertEqual(res, 'OK')
res = yield child1.get_process_parameters()
self.assertEqual(res, params)
params2 = {'more junk': 'my trunk'}
res = yield child1.set_process_parameters(params2)
self.assertEqual(res, 'OK')
res = yield child1.get_process_parameters()
ptest = dict(params)
ptest.update(params2)
self.assertEqual(res, ptest)
yield child1.shutdown()
def test_send(self):
pd1={'name':'consumer_number_1',
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',
'spawnargs':{'attach':self.queue1,\
'process parameters':{},
'delivery queues':{'queues':[self.queue2]}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
dmsg = DataMessageObject()
dmsg.notifcation = 'Junk'
dmsg.timestamp = pu.currenttime()
dmsg = dmsg.encode()
yield self.test_sup.send(self.queue1, 'data', dmsg)
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received', {})
sent = msg_cnt.get('sent', {})
self.assertEqual(sent.get(self.queue2), 1)
self.assertEqual(received.get(self.queue1), 1)
def test_create_topic(self):
# Create and Register a topic
topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")
# Make sure the topic starts out with blank stuff here...
self.assertEqual(topic.RegistryBranch,'master')
self.assertEqual(topic.RegistryIdentity,'')
self.assertEqual(topic.queue.type,'')
# Use the service to create a queue and register the topic
topic = yield self.pubsub.create_and_register_topic(topic)
# Make sure the queue properties were set.
self.assertEqual(topic.queue.type,'fanout')
#Spawn an baseconsumer and make sure a message is received on the new queue
pd1={'name':'consumer_number_1',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':[topic.queue.name]}}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
msg=DataMessageObject()
self.sup.send(topic.queue.name,'data',msg.encode())
yield pu.asleep(1)
dc1 = self._get_procinstance(child1_id)
#print 'DC1',dc1.receive_cnt
rec = dc1.receive_cnt[topic.queue.name]
self.assertEqual(rec,1)
def setUp(self):
yield self._start_container()
#self.sup = yield self._spawn_processes(services)
#Create a test queue
queue1 = dataobject.create_unique_identity()
queue_properties = {
queue1: {
'name_type': 'fanout',
'args': {
'scope': 'global'
}
}
}
yield bootstrap.declare_messaging(queue_properties)
self.queue1 = queue1
# Create a dataset to test with
ds = dap_tools.simple_grid_dataset()
fname = os.path.join(TEST_ARCHIVE_PATH, ds.name + '.nc')
self.fname_test = fname
fid = open(fname, "w")
fid.close()
# Create a persister process
pd1 = {
'name': 'persister_number_1',
'module': 'ion.services.dm.preservation.persister',
'procclass': 'PersisterConsumer',
'spawnargs': {
'attach': self.queue1,
'process parameters': {
'filename': fname
}
}
}
self.child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(self.child1)
# Don't do this - you can only get the instance in a test case -
# this is not a valid pattern in OTP
self.dc1 = self._get_procinstance(child1_id)
# Make sure it is up and working!
self.assertIn(self.queue1, self.dc1.dataReceivers)
self.ds = ds
self.fname = fname
def create_subscription(self, subscription):
'''
'''
subscription.create_new_reference()
subscription = yield self.create_consumer_args(subscription)
consumer_args = subscription.consumer_args
for name, args in consumer_args.items():
child = base_consumer.ConsumerDesc(**args)
child_id = yield self.spawn_child(child)
#subscription.consumer_procids[name]=child_id
subscription = yield self.reg.register(subscription)
defer.returnValue(subscription)
def test_spawn_attach_inst(self):
pd1={'name':'consumer_number_1', \
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
# NOT VALID IN OTP TO GET THE INSTANCE ONLY FOR TEST CASE
dc1 = self._get_procinstance(child1_id)
res = yield dc1.attach(self.queue1)
#@Todo Assert what?
self.assert_(res)
#self.assertEqual(child.proc_attached,self.queue1)
yield dc1.shutdown()
def test_spawn_attach_msg(self):
pd1={'name':'consumer_number_1', \
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
res = yield child1.attach(self.queue1)
self.assertEqual(res, 'OK')
#self.assertEqual(child.proc_attached,self.queue1)
res = yield child1.attach(None)
self.assertEqual(res, 'ERROR')
#self.assertEqual(child.proc_attached,None)
yield child1.shutdown()
def test_spawn_attach_args(self):
pd1 = {
'name': 'consumer_number_1',
'module':
'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass': 'ForwardingConsumer',
'spawnargs': {
'attach': [self.queue1]
}
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
# Don't do this - you can only get the instance in a test case -
# this is not a valid pattern in OTP
dc1 = self._get_procinstance(child1_id)
self.assertIn(self.queue1, dc1.dataReceivers)
def test_stream(self):
#Set up a logging consumer as a listenr
pd1 = {
'name': 'consumer_number_1',
'module':
'ion.services.dm.distribution.consumers.logging_consumer',
'procclass': 'LoggingConsumer',
'spawnargs': {
'attach': [self.queue1]
}
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
dc1 = self._get_procinstance(child1_id)
dsp1 = {
'name': 'data_stream_1',
'module': 'ion.services.dm.util.data_stream_producer',
'procclass': 'DataStreamProducer',
'spawnargs': {
'delivery queue': self.queue1,
'delivery interval': 5
}
}
child2 = ProcessDesc(**dsp1)
child2_id = yield self.test_sup.spawn_child(child2)
yield pu.asleep(2)
rec = dc1.receive_cnt[self.queue1]
self.assertEqual(rec, 1)
yield pu.asleep(5)
rec = dc1.receive_cnt[self.queue1]
self.assertEqual(rec, 2)
def test_exampleconsumer(self):
'''
@Brief Example Consumer is a demonstration of a more complex data consumer.
It uses DAP data messages and provides qaqc and event results on two
seperate queues.
'''
dpsc = DataPubsubClient(self.sup)
#Create and register 3 topics!
topic_raw = PubSubTopicResource.create("topic_raw","oceans, oil spill, fun things to do")
topic_raw = yield dpsc.define_topic(topic_raw)
#Create and register self.sup as a publisher
publisher = PublisherResource.create('Test Publisher', self.sup, topic_raw, 'DataObject')
publisher = yield dpsc.define_publisher(publisher)
logging.info('Defined Publisher: '+str(publisher))
# === Create a Consumer and queues - this will become part of define_subscription.
#Create two test queues - don't use topics to test the consumer
# To be replaced when the subscription service is ready
evt_queue=dataobject.create_unique_identity()
queue_properties = {evt_queue:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
pr_queue=dataobject.create_unique_identity()
queue_properties = {pr_queue:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
pd1={'name':'example_consumer_1',
'module':'ion.services.dm.distribution.consumers.example_consumer',
'procclass':'ExampleConsumer',
'spawnargs':{'attach':topic_raw.queue.name,\
'Process Parameters':{},\
'delivery queues':\
{'event_queue':evt_queue,\
'processed_queue':pr_queue}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
pd2={'name':'example_consumer_2',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':evt_queue,\
'Process Parameters':{}}\
}
child2 = base_consumer.ConsumerDesc(**pd2)
child2_id = yield self.test_sup.spawn_child(child2)
pd3={'name':'example_consumer_3',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':pr_queue,\
'Process Parameters':{}}\
}
child3 = base_consumer.ConsumerDesc(**pd3)
child3_id = yield self.test_sup.spawn_child(child3)
# === End of stuff that will be replaced with Subscription method...
# Create an example data message
dmsg = dap_tools.simple_datamessage(\
{'DataSet Name':'Simple Data','variables':\
{'time':{'long_name':'Data and Time','units':'seconds'},\
'height':{'long_name':'person height','units':'meters'}}}, \
{'time':(101,102,103,104,105,106,107,108,109,110), \
'height':(5,2,4,5,-1,9,3,888,3,4)})
result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg)
if result:
logging.info('Published Message')
else:
logging.info('Failed to Published Message')
# Need to await the delivery of data messages into the consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(evt_queue),2)
self.assertEqual(sent.get(pr_queue),1)
self.assertEqual(received.get(topic_raw.queue.name),1)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(evt_queue),2)
msg_cnt = yield child3.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(pr_queue),1)
# Publish a second message with different data
dmsg = dap_tools.simple_datamessage(\
{'DataSet Name':'Simple Data','variables':\
{'time':{'long_name':'Data and Time','units':'seconds'},\
'height':{'long_name':'person height','units':'meters'}}}, \
{'time':(111,112,123,114,115,116,117,118,119,120), \
'height':(8,986,4,-2,-1,5,3,1,4,5)})
result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg)
# Need to await the delivery of data messages into the consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(evt_queue),5)
self.assertEqual(sent.get(pr_queue),2)
self.assertEqual(received.get(topic_raw.queue.name),2)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(evt_queue),5)
msg_cnt = yield child3.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(pr_queue),2)
def test_pubsub(self):
dpsc = DataPubsubClient(self.sup)
# Create and Register a topic
topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")
topic = yield dpsc.define_topic(topic)
logging.info('Defined Topic: '+str(topic))
#Create and register self.sup as a publisher
publisher = PublisherResource.create('Test Publisher', self.sup, topic, 'DataObject')
publisher = yield dpsc.define_publisher(publisher)
logging.info('Defined Publisher: '+str(publisher))
# === Create a Consumer and queues - this will become part of define_subscription.
#Create two test queues - don't use topics to test the consumer
# To be replaced when the subscription service is ready
queue1=dataobject.create_unique_identity()
queue_properties = {queue1:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
queue2=dataobject.create_unique_identity()
queue_properties = {queue2:{'name_type':'fanout', 'args':{'scope':'global'}}}
yield bootstrap.declare_messaging(queue_properties)
pd1={'name':'example_consumer_1',
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',
'spawnargs':{'attach':topic.queue.name,\
'process parameters':{},\
'delivery queues':{'queues':[queue1,queue2]}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
# === End to be replaces with Define_Consumer
# Create and send a data message
data = {'Data':'in a dictionary'}
result = yield dpsc.publish(self.sup, topic.reference(), data)
if result:
logging.info('Published Message')
else:
logging.info('Failed to Published Message')
# Need to await the delivery of data messages into the (separate) consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(queue1),1)
self.assertEqual(sent.get(queue2),1)
self.assertEqual(received.get(topic.queue.name),1)
# === Create a Consumer - this will become part of define_subscription.
pd2={'name':'example_consumer_2',
'module':'ion.services.dm.distribution.consumers.logging_consumer',
'procclass':'LoggingConsumer',
'spawnargs':{'attach':queue1,\
'process parameters':{},\
'delivery queues':{}}\
}
child2 = base_consumer.ConsumerDesc(**pd2)
child2_id = yield self.test_sup.spawn_child(child2)
# === End of what will become part of the subscription definition
# Send the simple message again
result = yield dpsc.publish(self.sup, topic.reference(), data)
# Need to await the delivery of data messages into the (separate) consumers
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(queue1),2)
self.assertEqual(sent.get(queue2),2)
self.assertEqual(received.get(topic.queue.name),2)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent,{})
self.assertEqual(received.get(queue1),1)
