def test_send(self):
pd1={'name':'consumer_number_1',
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',
'spawnargs':{'attach':self.queue1,\
'process parameters':{},
'delivery queues':{'queues':[self.queue2]}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
dmsg = DataMessageObject()
dmsg.notifcation = 'Junk'
dmsg.timestamp = pu.currenttime()
dmsg = dmsg.encode()
yield self.test_sup.send(self.queue1, 'data', dmsg)
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(self.queue2),1)
self.assertEqual(received.get(self.queue1),1)
def ondata(self, data, notification, timestamp, queue='', max_points=15, **kwargs):
vals = data.split(',')
print 'VALS:',vals
description = [('v1','number', 'value1'),
('v2','number', 'value2'),
('v3','number', 'value3'),
('v4','number', 'value4')
]
self.pdata.append([float(vals[0]),float(vals[1]),float(vals[2]),float(vals[3])])
dlen = len(self.pdata)
if dlen > max_points:
self.pdata = self.pdata[dlen-max_points : ]
data_table = gviz_api.DataTable(description)
data_table.LoadData(self.pdata)
#json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
json = data_table.ToJSon()
# Make message for the screen below
msg = '<p>Timestamp: %s </p>\n' % pu.currenttime()
page = line_template % {'msg':msg,'json':json}
self.queue_result(queue,page,'Google Viz of message counts')
def test_send(self):
pd1={'name':'consumer_number_1',
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',
'spawnargs':{'attach':self.queue1,\
'process parameters':{},
'delivery queues':{'queues':[self.queue2]}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
dmsg = DataMessageObject()
dmsg.notifcation = 'Junk'
dmsg.timestamp = pu.currenttime()
dmsg = dmsg.encode()
yield self.test_sup.send(self.queue1, 'data', dmsg)
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received', {})
sent = msg_cnt.get('sent', {})
self.assertEqual(sent.get(self.queue2), 1)
self.assertEqual(received.get(self.queue1), 1)
def ondata(self,
data,
notification,
timestamp,
queue='',
max_points=15,
**kwargs):
vals = data.split(',')
print 'VALS:', vals
description = [('v1', 'number', 'value1'), ('v2', 'number', 'value2'),
('v3', 'number', 'value3'), ('v4', 'number', 'value4')]
self.pdata.append(
[float(vals[0]),
float(vals[1]),
float(vals[2]),
float(vals[3])])
dlen = len(self.pdata)
if dlen > max_points:
self.pdata = self.pdata[dlen - max_points:]
data_table = gviz_api.DataTable(description)
data_table.LoadData(self.pdata)
#json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
json = data_table.ToJSon()
# Make message for the screen below
msg = '<p>Timestamp: %s </p>\n' % pu.currenttime()
page = line_template % {'msg': msg, 'json': json}
self.queue_result(queue, page, 'Google Viz of message counts')
def op_publish(self, content, headers, msg):
"""Service operation: Publish data message on a topic
"""
logging.debug(self.__class__.__name__ + ', op_' + headers['op'] +
' Received: ' + str(headers))
publication = dataobject.Resource.decode(content)
logging.info(self.__class__.__name__ + ' recieved: op_' +
headers['op'] + ', publication')
#logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication: \n' + str(publication))
#Get the data
data = publication.data
# Get the Topic
topic_ref = publication.topic_ref
topic = yield self.reg.get(topic_ref.reference(head=True))
if not topic:
logging.info(self.__class__.__name__ + ' recieved: op_' +
headers['op'] + ', topic invalid!')
yield self.reply_err(msg, 'Topic does not exist')
return
#Check publisher is valid!
publisher = dm_resource_descriptions.PublisherResource()
publisher.publisher = publication.publisher
# Get the publications which this process is registered fro
reg_pubs = yield self.reg.find(publisher,
regex=False,
attnames=['publisher'])
valid = False
for pub in reg_pubs:
if topic.reference(head=True) in pub.topics:
valid = True
logging.debug(self.__class__.__name__ +
'; Publishing to topic: \n' + str(topic))
break
if not valid:
logging.info(self.__class__.__name__ + ' recieved: op_' +
headers['op'] +
', publisher not registered for topic!')
yield self.reply_err(msg, 'Publisher not registered for topic!')
return
if not data.notification:
data.notification = 'Data topic: ' + topic.name
data.timestamp = pu.currenttime()
# Todo: impersonate message as from sender -
#@Todo - Move the actual publish command back to the client side!
yield self.send(topic.queue.name, 'data', data.encode(), {})
logging.info(self.__class__.__name__ + ': op_' + headers['op'] +
' Success!')
yield self.reply_ok(msg, '')
def test_send_chain(self):
pd1={'name':'consumer_number_1',
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',
'spawnargs':{'attach':self.queue1,\
'process parameters':{},\
'delivery queues':{'queues':[self.queue2]}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
dmsg = DataMessageObject()
dmsg.notifcation = 'Junk'
dmsg.timestamp = pu.currenttime()
dmsg = dmsg.encode()
yield self.test_sup.send(self.queue1, 'data', dmsg)
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received', {})
sent = msg_cnt.get('sent', {})
self.assertEqual(sent.get(self.queue2), 1)
self.assertEqual(received.get(self.queue1), 1)
#Spawn another process to listen to queue 2
pd2={'name':'consumer_number_2', \
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',\
'spawnargs':{'attach':self.queue2}}
child2 = base_consumer.ConsumerDesc(**pd2)
child2_id = yield self.test_sup.spawn_child(child2)
# Tell the first consumer to pass results to the second!
#res = yield child1.set_params({'queues':[self.queue2]})
yield self.test_sup.send(self.queue1, 'data', dmsg)
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received', {})
sent = msg_cnt.get('sent', {})
self.assertEqual(sent.get(self.queue2), 2)
self.assertEqual(received.get(self.queue1), 2)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received', {})
sent = msg_cnt.get('sent')
self.assertEqual(sent, {})
self.assertEqual(received.get(self.queue2), 1)
yield child1.shutdown()
yield child2.shutdown()
def op_publish(self, content, headers, msg):
"""Service operation: Publish data message on a topic
"""
logging.debug(self.__class__.__name__ +', op_'+ headers['op'] +' Received: ' + str(headers))
publication = dataobject.Resource.decode(content)
logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication')
#logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication: \n' + str(publication))
#Get the data
data = publication.data
# Get the Topic
topic_ref = publication.topic_ref
topic = yield self.reg.get(topic_ref.reference(head=True))
if not topic:
logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', topic invalid!')
yield self.reply_err(msg, 'Topic does not exist')
return
#Check publisher is valid!
publisher = dm_resource_descriptions.PublisherResource()
publisher.publisher = publication.publisher
# Get the publications which this process is registered fro
reg_pubs = yield self.reg.find(publisher, regex=False, attnames=['publisher'])
valid = False
for pub in reg_pubs:
if topic.reference(head=True) in pub.topics:
valid = True
logging.debug(self.__class__.__name__ + '; Publishing to topic: \n' + str(topic))
break
if not valid:
logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publisher not registered for topic!')
yield self.reply_err(msg, 'Publisher not registered for topic!')
return
if not data.notification:
data.notification = 'Data topic: ' + topic.name
data.timestamp = pu.currenttime()
# Todo: impersonate message as from sender -
#@Todo - Move the actual publish command back to the client side!
yield self.send(topic.queue.name, 'data', data.encode(), {})
logging.info(self.__class__.__name__ + ': op_'+ headers['op'] + ' Success!')
yield self.reply_ok(msg, '')
def onschedule(self, queue='', max_points=50, **kwargs):
'''
This method is called when it is time to actually send the results
in this case it is not needed, but must be over-riden...
'''
# Count the messages recieved
total = 0
for k,v in self.receive_cnt.items():
total += v
# Count the messages this interval
interval = 0
for k,v in self.interval_cnt.items():
interval += v
notification = '''Message Counter has received %s messages, %s since last report''' \
% (total, interval)
description = [('interval#','string', 'Interval Number'),
('interval','number','Messages this Interval'),
('total','number','Total Messages')]
self.interval_number +=1
self.data.append([str(self.interval_number),interval,total])
dlen = len(self.data)
if dlen > max_points:
self.data = self.data[dlen-max_points : ]
data_table = gviz_api.DataTable(description)
data_table.LoadData(self.data)
#json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
json = data_table.ToJSon()
# Make message for the screen below
msg = '<p>Number of messages received so far by Web Viz Consumer: %s </p>\n' % total
msg += '<p>Timestamp: %s </p>\n' % pu.currenttime()
page = line_template % {'msg':msg,'json':json}
self.queue_result(queue,page,'Google Viz of message counts')
def digest(self):
logging.info(self.__class__.__name__ +"; Digesting results!")
# Stop the loop if it is running - start again when next data is received
if self.loop_running:
#self.loop.stop()
self.loop_running = False
args = dict(self.params)
args.update(self.deliver)
yield defer.maybeDeferred(self.onschedule, **args)
yield self.deliver_messages()
# Update last_delivered
self.interval_cnt={} # Reset the interval receive count
self.last_delivered = pu.currenttime()
logging.info(self.__class__.__name__ +"; digest: Finished sending results")
def onschedule(self, queue='', max_points=50, **kwargs):
'''
This method is called when it is time to actually send the results
in this case it is not needed, but must be over-riden...
'''
# Count the messages recieved
total = 0
for k, v in self.receive_cnt.items():
total += v
# Count the messages this interval
interval = 0
for k, v in self.interval_cnt.items():
interval += v
notification = '''Message Counter has received %s messages, %s since last report''' \
% (total, interval)
description = [('interval#', 'string', 'Interval Number'),
('interval', 'number', 'Messages this Interval'),
('total', 'number', 'Total Messages')]
self.interval_number += 1
self.data.append([str(self.interval_number), interval, total])
dlen = len(self.data)
if dlen > max_points:
self.data = self.data[dlen - max_points:]
data_table = gviz_api.DataTable(description)
data_table.LoadData(self.data)
#json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
json = data_table.ToJSon()
# Make message for the screen below
msg = '<p>Number of messages received so far by Web Viz Consumer: %s </p>\n' % total
msg += '<p>Timestamp: %s </p>\n' % pu.currenttime()
page = line_template % {'msg': msg, 'json': json}
self.queue_result(queue, page, 'Google Viz of message counts')
def queue_result(self,queue, data=None, notification=''):
if isinstance(data, DatasetType):
msg = dap_tools.ds2dap_msg(data)
elif isinstance(data, str):
msg = StringMessageObject()
msg.data=data
elif isinstance(data, dict):
msg = DictionaryMessageObject()
msg.data=data
elif not data:
msg = DataMessageObject()
else:
raise RuntimeError('Invalid data type passed to send_result in class %s: type:' % (self.__class__.__name__, type(data)))
if not notification:
notification = self.__class__.__name__ + ' received a message!'
msg.notification = notification
msg.timestamp = pu.currenttime()
self.msgs_to_send.append((queue, msg.encode()))
def ondata(self, data, notification, timestamp, queue='', max_points=50, **kwargs):
gname = data.keys()
grid = data[gname[0]]
for name in grid.keys():
if not name == 'time':
vname = name
if not 'time' in grid.keys():
raise RuntimeError('No axis time in grid dataset')
if not len(grid.keys()) == 2:
raise RuntimeError('Only time series are allowed in this consumer!')
description = [('time','string', 'time (Seconds)'),
(vname,'number', vname)]
vals = zip(grid['time'], grid[vname])
for val in vals:
self.pdata.append([str(val[0]),float(val[1])])
dlen = len(self.pdata)
if dlen > max_points:
self.pdata = self.pdata[dlen-max_points : ]
data_table = gviz_api.DataTable(description)
data_table.LoadData(self.pdata)
#json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
json = data_table.ToJSon()
# Make message for the screen below
msg = '<p>Timestamp: %s </p>\n' % pu.currenttime()
page = line_template % {'msg':msg,'json':json}
self.queue_result(queue,page,'Google Viz of message counts')
def op_data(self, content, headers, msg):
logging.debug(self.__class__.__name__ +', MSG Received: ' + str(headers))
logging.info(self.__class__.__name__ + '; Calling data process!')
# Keep a record of messages received
#@Note this could get big! What todo?
self.receive_cnt[headers.get('receiver')] += 1
#self.received_msg.append(content) # Do not keep the messages!
# Unpack the message and turn it into data
datamessage = dataobject.DataObject.decode(content)
if isinstance(datamessage, DAPMessageObject):
data = dap_tools.dap_msg2ds(datamessage)
elif isinstance(datamessage, (StringMessageObject, DictionaryMessageObject)):
data = datamessage.data
else:
data = None
notification = datamessage.notification
timestamp = datamessage.timestamp
# Build the keyword args for ondata
args = dict(self.params)
args.update(self.deliver)
logging.debug('**ARGS to ondata:'+str(args))
yield defer.maybeDeferred(self.ondata, data, notification, timestamp, **args)
logging.info(self.__class__.__name__ +"; op_data: Finished data processing")
# Is this a consumer with digest delivery?
if not self.delivery_interval:
# if not send the messages from ondata...
yield self.deliver_messages()
logging.info(self.__class__.__name__ +"; op_data: Finished sending results")
else: # Do the digets thing...
if self.interval_cnt.has_key(headers.get('receiver')):
self.interval_cnt[headers.get('receiver')] += 1
else:
self.interval_cnt[headers.get('receiver')] = 1
logging.debug(self.__class__.__name__ +"; op_data: digest state: \n" + \
"Last Delivered: " +str(self.last_delivered) +";\n" +\
"Loop Running: " +str(self.loop_running))
# First time data has arrived?
if self.last_delivered == None:
self.last_delivered = pu.currenttime()
if not self.loop_running:
# Is it already time to go?
if self.last_delivered + self.delivery_interval <= pu.currenttime():
yield self.digest()
# if data has arrived but it is not yet time to deliver, schedule a call back
else:
self.loop_running = True
delta_t = self.last_delivered + self.delivery_interval - pu.currenttime()
logging.debug('Scheduling a call back in %s seconds' % delta_t)
#self.loop.start(delta_t)
reactor.callLater(delta_t, self.digest)
def test_send_chain(self):
pd1={'name':'consumer_number_1',
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',
'spawnargs':{'attach':self.queue1,\
'process parameters':{},\
'delivery queues':{'queues':[self.queue2]}}\
}
child1 = base_consumer.ConsumerDesc(**pd1)
child1_id = yield self.test_sup.spawn_child(child1)
dmsg = DataMessageObject()
dmsg.notifcation = 'Junk'
dmsg.timestamp = pu.currenttime()
dmsg = dmsg.encode()
yield self.test_sup.send(self.queue1, 'data', dmsg)
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(self.queue2),1)
self.assertEqual(received.get(self.queue1),1)
#Spawn another process to listen to queue 2
pd2={'name':'consumer_number_2', \
'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
'procclass':'ForwardingConsumer',\
'spawnargs':{'attach':self.queue2}}
child2 = base_consumer.ConsumerDesc(**pd2)
child2_id = yield self.test_sup.spawn_child(child2)
# Tell the first consumer to pass results to the second!
#res = yield child1.set_params({'queues':[self.queue2]})
yield self.test_sup.send(self.queue1, 'data', dmsg)
yield pu.asleep(1)
msg_cnt = yield child1.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent',{})
self.assertEqual(sent.get(self.queue2),2)
self.assertEqual(received.get(self.queue1),2)
msg_cnt = yield child2.get_msg_count()
received = msg_cnt.get('received',{})
sent = msg_cnt.get('sent')
self.assertEqual(sent,{})
self.assertEqual(received.get(self.queue2),1)
yield child1.shutdown()
yield child2.shutdown()