def test_send(self): pd1={'name':'consumer_number_1', 'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'procclass':'ForwardingConsumer', 'spawnargs':{'attach':self.queue1,\ 'process parameters':{}, 'delivery queues':{'queues':[self.queue2]}}\ } child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) dmsg = DataMessageObject() dmsg.notifcation = 'Junk' dmsg.timestamp = pu.currenttime() dmsg = dmsg.encode() yield self.test_sup.send(self.queue1, 'data', dmsg) yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(self.queue2),1) self.assertEqual(received.get(self.queue1),1)
def ondata(self, data, notification, timestamp, queue='', max_points=15, **kwargs): vals = data.split(',') print 'VALS:',vals description = [('v1','number', 'value1'), ('v2','number', 'value2'), ('v3','number', 'value3'), ('v4','number', 'value4') ] self.pdata.append([float(vals[0]),float(vals[1]),float(vals[2]),float(vals[3])]) dlen = len(self.pdata) if dlen > max_points: self.pdata = self.pdata[dlen-max_points : ] data_table = gviz_api.DataTable(description) data_table.LoadData(self.pdata) #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary") json = data_table.ToJSon() # Make message for the screen below msg = '<p>Timestamp: %s </p>\n' % pu.currenttime() page = line_template % {'msg':msg,'json':json} self.queue_result(queue,page,'Google Viz of message counts')
def test_send(self): pd1={'name':'consumer_number_1', 'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'procclass':'ForwardingConsumer', 'spawnargs':{'attach':self.queue1,\ 'process parameters':{}, 'delivery queues':{'queues':[self.queue2]}}\ } child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) dmsg = DataMessageObject() dmsg.notifcation = 'Junk' dmsg.timestamp = pu.currenttime() dmsg = dmsg.encode() yield self.test_sup.send(self.queue1, 'data', dmsg) yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received', {}) sent = msg_cnt.get('sent', {}) self.assertEqual(sent.get(self.queue2), 1) self.assertEqual(received.get(self.queue1), 1)
def ondata(self, data, notification, timestamp, queue='', max_points=15, **kwargs): vals = data.split(',') print 'VALS:', vals description = [('v1', 'number', 'value1'), ('v2', 'number', 'value2'), ('v3', 'number', 'value3'), ('v4', 'number', 'value4')] self.pdata.append( [float(vals[0]), float(vals[1]), float(vals[2]), float(vals[3])]) dlen = len(self.pdata) if dlen > max_points: self.pdata = self.pdata[dlen - max_points:] data_table = gviz_api.DataTable(description) data_table.LoadData(self.pdata) #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary") json = data_table.ToJSon() # Make message for the screen below msg = '<p>Timestamp: %s </p>\n' % pu.currenttime() page = line_template % {'msg': msg, 'json': json} self.queue_result(queue, page, 'Google Viz of message counts')
def op_publish(self, content, headers, msg): """Service operation: Publish data message on a topic """ logging.debug(self.__class__.__name__ + ', op_' + headers['op'] + ' Received: ' + str(headers)) publication = dataobject.Resource.decode(content) logging.info(self.__class__.__name__ + ' recieved: op_' + headers['op'] + ', publication') #logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication: \n' + str(publication)) #Get the data data = publication.data # Get the Topic topic_ref = publication.topic_ref topic = yield self.reg.get(topic_ref.reference(head=True)) if not topic: logging.info(self.__class__.__name__ + ' recieved: op_' + headers['op'] + ', topic invalid!') yield self.reply_err(msg, 'Topic does not exist') return #Check publisher is valid! publisher = dm_resource_descriptions.PublisherResource() publisher.publisher = publication.publisher # Get the publications which this process is registered fro reg_pubs = yield self.reg.find(publisher, regex=False, attnames=['publisher']) valid = False for pub in reg_pubs: if topic.reference(head=True) in pub.topics: valid = True logging.debug(self.__class__.__name__ + '; Publishing to topic: \n' + str(topic)) break if not valid: logging.info(self.__class__.__name__ + ' recieved: op_' + headers['op'] + ', publisher not registered for topic!') yield self.reply_err(msg, 'Publisher not registered for topic!') return if not data.notification: data.notification = 'Data topic: ' + topic.name data.timestamp = pu.currenttime() # Todo: impersonate message as from sender - #@Todo - Move the actual publish command back to the client side! yield self.send(topic.queue.name, 'data', data.encode(), {}) logging.info(self.__class__.__name__ + ': op_' + headers['op'] + ' Success!') yield self.reply_ok(msg, '')
def test_send_chain(self): pd1={'name':'consumer_number_1', 'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'procclass':'ForwardingConsumer', 'spawnargs':{'attach':self.queue1,\ 'process parameters':{},\ 'delivery queues':{'queues':[self.queue2]}}\ } child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) dmsg = DataMessageObject() dmsg.notifcation = 'Junk' dmsg.timestamp = pu.currenttime() dmsg = dmsg.encode() yield self.test_sup.send(self.queue1, 'data', dmsg) yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received', {}) sent = msg_cnt.get('sent', {}) self.assertEqual(sent.get(self.queue2), 1) self.assertEqual(received.get(self.queue1), 1) #Spawn another process to listen to queue 2 pd2={'name':'consumer_number_2', \ 'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'procclass':'ForwardingConsumer',\ 'spawnargs':{'attach':self.queue2}} child2 = base_consumer.ConsumerDesc(**pd2) child2_id = yield self.test_sup.spawn_child(child2) # Tell the first consumer to pass results to the second! #res = yield child1.set_params({'queues':[self.queue2]}) yield self.test_sup.send(self.queue1, 'data', dmsg) yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received', {}) sent = msg_cnt.get('sent', {}) self.assertEqual(sent.get(self.queue2), 2) self.assertEqual(received.get(self.queue1), 2) msg_cnt = yield child2.get_msg_count() received = msg_cnt.get('received', {}) sent = msg_cnt.get('sent') self.assertEqual(sent, {}) self.assertEqual(received.get(self.queue2), 1) yield child1.shutdown() yield child2.shutdown()
def op_publish(self, content, headers, msg): """Service operation: Publish data message on a topic """ logging.debug(self.__class__.__name__ +', op_'+ headers['op'] +' Received: ' + str(headers)) publication = dataobject.Resource.decode(content) logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication') #logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication: \n' + str(publication)) #Get the data data = publication.data # Get the Topic topic_ref = publication.topic_ref topic = yield self.reg.get(topic_ref.reference(head=True)) if not topic: logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', topic invalid!') yield self.reply_err(msg, 'Topic does not exist') return #Check publisher is valid! publisher = dm_resource_descriptions.PublisherResource() publisher.publisher = publication.publisher # Get the publications which this process is registered fro reg_pubs = yield self.reg.find(publisher, regex=False, attnames=['publisher']) valid = False for pub in reg_pubs: if topic.reference(head=True) in pub.topics: valid = True logging.debug(self.__class__.__name__ + '; Publishing to topic: \n' + str(topic)) break if not valid: logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publisher not registered for topic!') yield self.reply_err(msg, 'Publisher not registered for topic!') return if not data.notification: data.notification = 'Data topic: ' + topic.name data.timestamp = pu.currenttime() # Todo: impersonate message as from sender - #@Todo - Move the actual publish command back to the client side! yield self.send(topic.queue.name, 'data', data.encode(), {}) logging.info(self.__class__.__name__ + ': op_'+ headers['op'] + ' Success!') yield self.reply_ok(msg, '')
def onschedule(self, queue='', max_points=50, **kwargs): ''' This method is called when it is time to actually send the results in this case it is not needed, but must be over-riden... ''' # Count the messages recieved total = 0 for k,v in self.receive_cnt.items(): total += v # Count the messages this interval interval = 0 for k,v in self.interval_cnt.items(): interval += v notification = '''Message Counter has received %s messages, %s since last report''' \ % (total, interval) description = [('interval#','string', 'Interval Number'), ('interval','number','Messages this Interval'), ('total','number','Total Messages')] self.interval_number +=1 self.data.append([str(self.interval_number),interval,total]) dlen = len(self.data) if dlen > max_points: self.data = self.data[dlen-max_points : ] data_table = gviz_api.DataTable(description) data_table.LoadData(self.data) #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary") json = data_table.ToJSon() # Make message for the screen below msg = '<p>Number of messages received so far by Web Viz Consumer: %s </p>\n' % total msg += '<p>Timestamp: %s </p>\n' % pu.currenttime() page = line_template % {'msg':msg,'json':json} self.queue_result(queue,page,'Google Viz of message counts')
def digest(self): logging.info(self.__class__.__name__ +"; Digesting results!") # Stop the loop if it is running - start again when next data is received if self.loop_running: #self.loop.stop() self.loop_running = False args = dict(self.params) args.update(self.deliver) yield defer.maybeDeferred(self.onschedule, **args) yield self.deliver_messages() # Update last_delivered self.interval_cnt={} # Reset the interval receive count self.last_delivered = pu.currenttime() logging.info(self.__class__.__name__ +"; digest: Finished sending results")
def onschedule(self, queue='', max_points=50, **kwargs): ''' This method is called when it is time to actually send the results in this case it is not needed, but must be over-riden... ''' # Count the messages recieved total = 0 for k, v in self.receive_cnt.items(): total += v # Count the messages this interval interval = 0 for k, v in self.interval_cnt.items(): interval += v notification = '''Message Counter has received %s messages, %s since last report''' \ % (total, interval) description = [('interval#', 'string', 'Interval Number'), ('interval', 'number', 'Messages this Interval'), ('total', 'number', 'Total Messages')] self.interval_number += 1 self.data.append([str(self.interval_number), interval, total]) dlen = len(self.data) if dlen > max_points: self.data = self.data[dlen - max_points:] data_table = gviz_api.DataTable(description) data_table.LoadData(self.data) #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary") json = data_table.ToJSon() # Make message for the screen below msg = '<p>Number of messages received so far by Web Viz Consumer: %s </p>\n' % total msg += '<p>Timestamp: %s </p>\n' % pu.currenttime() page = line_template % {'msg': msg, 'json': json} self.queue_result(queue, page, 'Google Viz of message counts')
def queue_result(self,queue, data=None, notification=''): if isinstance(data, DatasetType): msg = dap_tools.ds2dap_msg(data) elif isinstance(data, str): msg = StringMessageObject() msg.data=data elif isinstance(data, dict): msg = DictionaryMessageObject() msg.data=data elif not data: msg = DataMessageObject() else: raise RuntimeError('Invalid data type passed to send_result in class %s: type:' % (self.__class__.__name__, type(data))) if not notification: notification = self.__class__.__name__ + ' received a message!' msg.notification = notification msg.timestamp = pu.currenttime() self.msgs_to_send.append((queue, msg.encode()))
def ondata(self, data, notification, timestamp, queue='', max_points=50, **kwargs): gname = data.keys() grid = data[gname[0]] for name in grid.keys(): if not name == 'time': vname = name if not 'time' in grid.keys(): raise RuntimeError('No axis time in grid dataset') if not len(grid.keys()) == 2: raise RuntimeError('Only time series are allowed in this consumer!') description = [('time','string', 'time (Seconds)'), (vname,'number', vname)] vals = zip(grid['time'], grid[vname]) for val in vals: self.pdata.append([str(val[0]),float(val[1])]) dlen = len(self.pdata) if dlen > max_points: self.pdata = self.pdata[dlen-max_points : ] data_table = gviz_api.DataTable(description) data_table.LoadData(self.pdata) #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary") json = data_table.ToJSon() # Make message for the screen below msg = '<p>Timestamp: %s </p>\n' % pu.currenttime() page = line_template % {'msg':msg,'json':json} self.queue_result(queue,page,'Google Viz of message counts')
def op_data(self, content, headers, msg): logging.debug(self.__class__.__name__ +', MSG Received: ' + str(headers)) logging.info(self.__class__.__name__ + '; Calling data process!') # Keep a record of messages received #@Note this could get big! What todo? self.receive_cnt[headers.get('receiver')] += 1 #self.received_msg.append(content) # Do not keep the messages! # Unpack the message and turn it into data datamessage = dataobject.DataObject.decode(content) if isinstance(datamessage, DAPMessageObject): data = dap_tools.dap_msg2ds(datamessage) elif isinstance(datamessage, (StringMessageObject, DictionaryMessageObject)): data = datamessage.data else: data = None notification = datamessage.notification timestamp = datamessage.timestamp # Build the keyword args for ondata args = dict(self.params) args.update(self.deliver) logging.debug('**ARGS to ondata:'+str(args)) yield defer.maybeDeferred(self.ondata, data, notification, timestamp, **args) logging.info(self.__class__.__name__ +"; op_data: Finished data processing") # Is this a consumer with digest delivery? if not self.delivery_interval: # if not send the messages from ondata... yield self.deliver_messages() logging.info(self.__class__.__name__ +"; op_data: Finished sending results") else: # Do the digets thing... if self.interval_cnt.has_key(headers.get('receiver')): self.interval_cnt[headers.get('receiver')] += 1 else: self.interval_cnt[headers.get('receiver')] = 1 logging.debug(self.__class__.__name__ +"; op_data: digest state: \n" + \ "Last Delivered: " +str(self.last_delivered) +";\n" +\ "Loop Running: " +str(self.loop_running)) # First time data has arrived? if self.last_delivered == None: self.last_delivered = pu.currenttime() if not self.loop_running: # Is it already time to go? if self.last_delivered + self.delivery_interval <= pu.currenttime(): yield self.digest() # if data has arrived but it is not yet time to deliver, schedule a call back else: self.loop_running = True delta_t = self.last_delivered + self.delivery_interval - pu.currenttime() logging.debug('Scheduling a call back in %s seconds' % delta_t) #self.loop.start(delta_t) reactor.callLater(delta_t, self.digest)
def test_send_chain(self): pd1={'name':'consumer_number_1', 'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'procclass':'ForwardingConsumer', 'spawnargs':{'attach':self.queue1,\ 'process parameters':{},\ 'delivery queues':{'queues':[self.queue2]}}\ } child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) dmsg = DataMessageObject() dmsg.notifcation = 'Junk' dmsg.timestamp = pu.currenttime() dmsg = dmsg.encode() yield self.test_sup.send(self.queue1, 'data', dmsg) yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(self.queue2),1) self.assertEqual(received.get(self.queue1),1) #Spawn another process to listen to queue 2 pd2={'name':'consumer_number_2', \ 'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'procclass':'ForwardingConsumer',\ 'spawnargs':{'attach':self.queue2}} child2 = base_consumer.ConsumerDesc(**pd2) child2_id = yield self.test_sup.spawn_child(child2) # Tell the first consumer to pass results to the second! #res = yield child1.set_params({'queues':[self.queue2]}) yield self.test_sup.send(self.queue1, 'data', dmsg) yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(self.queue2),2) self.assertEqual(received.get(self.queue1),2) msg_cnt = yield child2.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent') self.assertEqual(sent,{}) self.assertEqual(received.get(self.queue2),1) yield child1.shutdown() yield child2.shutdown()