def test_send(self):
        pd1={'name':'consumer_number_1',
                 'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
                 'procclass':'ForwardingConsumer',
                 'spawnargs':{'attach':self.queue1,\
                              'process parameters':{},
                              'delivery queues':{'queues':[self.queue2]}}\
                    }
        child1 = base_consumer.ConsumerDesc(**pd1)

        child1_id = yield self.test_sup.spawn_child(child1)

        dmsg = DataMessageObject()
        dmsg.notifcation = 'Junk'
        dmsg.timestamp = pu.currenttime()
        dmsg = dmsg.encode()
        
        yield self.test_sup.send(self.queue1, 'data', dmsg)
        
        yield pu.asleep(1)
        msg_cnt = yield child1.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent.get(self.queue2),1)
        self.assertEqual(received.get(self.queue1),1)
 def ondata(self, data, notification, timestamp, queue='', max_points=15, **kwargs):
     
     vals = data.split(',')
     print 'VALS:',vals
     
     description = [('v1','number', 'value1'),
                     ('v2','number', 'value2'),
                     ('v3','number', 'value3'),
                     ('v4','number', 'value4')
                    ]
     
     self.pdata.append([float(vals[0]),float(vals[1]),float(vals[2]),float(vals[3])])
     
     dlen = len(self.pdata)
     if dlen > max_points:
         self.pdata = self.pdata[dlen-max_points : ]
     
     data_table = gviz_api.DataTable(description)
     data_table.LoadData(self.pdata)
     #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
     json = data_table.ToJSon()
         
     # Make message for the screen below
     msg = '<p>Timestamp: %s </p>\n' % pu.currenttime()
     
     page = line_template % {'msg':msg,'json':json}
         
     self.queue_result(queue,page,'Google Viz of message counts')
Beispiel #3
0
    def test_send(self):
        pd1={'name':'consumer_number_1',
                 'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
                 'procclass':'ForwardingConsumer',
                 'spawnargs':{'attach':self.queue1,\
                              'process parameters':{},
                              'delivery queues':{'queues':[self.queue2]}}\
                    }
        child1 = base_consumer.ConsumerDesc(**pd1)

        child1_id = yield self.test_sup.spawn_child(child1)

        dmsg = DataMessageObject()
        dmsg.notifcation = 'Junk'
        dmsg.timestamp = pu.currenttime()
        dmsg = dmsg.encode()

        yield self.test_sup.send(self.queue1, 'data', dmsg)

        yield pu.asleep(1)
        msg_cnt = yield child1.get_msg_count()
        received = msg_cnt.get('received', {})
        sent = msg_cnt.get('sent', {})
        self.assertEqual(sent.get(self.queue2), 1)
        self.assertEqual(received.get(self.queue1), 1)
    def ondata(self,
               data,
               notification,
               timestamp,
               queue='',
               max_points=15,
               **kwargs):

        vals = data.split(',')
        print 'VALS:', vals

        description = [('v1', 'number', 'value1'), ('v2', 'number', 'value2'),
                       ('v3', 'number', 'value3'), ('v4', 'number', 'value4')]

        self.pdata.append(
            [float(vals[0]),
             float(vals[1]),
             float(vals[2]),
             float(vals[3])])

        dlen = len(self.pdata)
        if dlen > max_points:
            self.pdata = self.pdata[dlen - max_points:]

        data_table = gviz_api.DataTable(description)
        data_table.LoadData(self.pdata)
        #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
        json = data_table.ToJSon()

        # Make message for the screen below
        msg = '<p>Timestamp: %s </p>\n' % pu.currenttime()

        page = line_template % {'msg': msg, 'json': json}

        self.queue_result(queue, page, 'Google Viz of message counts')
    def op_publish(self, content, headers, msg):
        """Service operation: Publish data message on a topic
        """
        logging.debug(self.__class__.__name__ + ', op_' + headers['op'] +
                      ' Received: ' + str(headers))
        publication = dataobject.Resource.decode(content)
        logging.info(self.__class__.__name__ + ' recieved: op_' +
                     headers['op'] + ', publication')
        #logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication: \n' + str(publication))

        #Get the data
        data = publication.data

        # Get the Topic
        topic_ref = publication.topic_ref
        topic = yield self.reg.get(topic_ref.reference(head=True))
        if not topic:
            logging.info(self.__class__.__name__ + ' recieved: op_' +
                         headers['op'] + ', topic invalid!')
            yield self.reply_err(msg, 'Topic does not exist')
            return

        #Check publisher is valid!
        publisher = dm_resource_descriptions.PublisherResource()
        publisher.publisher = publication.publisher

        # Get the publications which this process is registered fro
        reg_pubs = yield self.reg.find(publisher,
                                       regex=False,
                                       attnames=['publisher'])

        valid = False
        for pub in reg_pubs:

            if topic.reference(head=True) in pub.topics:
                valid = True
                logging.debug(self.__class__.__name__ +
                              '; Publishing to topic: \n' + str(topic))
                break

        if not valid:
            logging.info(self.__class__.__name__ + ' recieved: op_' +
                         headers['op'] +
                         ', publisher not registered for topic!')
            yield self.reply_err(msg, 'Publisher not registered for topic!')
            return

        if not data.notification:
            data.notification = 'Data topic: ' + topic.name

        data.timestamp = pu.currenttime()

        # Todo: impersonate message as from sender -
        #@Todo - Move the actual publish command back to the client side!
        yield self.send(topic.queue.name, 'data', data.encode(), {})

        logging.info(self.__class__.__name__ + ': op_' + headers['op'] +
                     ' Success!')
        yield self.reply_ok(msg, '')
Beispiel #6
0
    def test_send_chain(self):
        pd1={'name':'consumer_number_1',
                 'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
                 'procclass':'ForwardingConsumer',
                 'spawnargs':{'attach':self.queue1,\
                              'process parameters':{},\
                              'delivery queues':{'queues':[self.queue2]}}\
                    }
        child1 = base_consumer.ConsumerDesc(**pd1)

        child1_id = yield self.test_sup.spawn_child(child1)

        dmsg = DataMessageObject()
        dmsg.notifcation = 'Junk'
        dmsg.timestamp = pu.currenttime()
        dmsg = dmsg.encode()

        yield self.test_sup.send(self.queue1, 'data', dmsg)

        yield pu.asleep(1)

        msg_cnt = yield child1.get_msg_count()
        received = msg_cnt.get('received', {})
        sent = msg_cnt.get('sent', {})
        self.assertEqual(sent.get(self.queue2), 1)
        self.assertEqual(received.get(self.queue1), 1)

        #Spawn another process to listen to queue 2
        pd2={'name':'consumer_number_2', \
                 'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
                 'procclass':'ForwardingConsumer',\
                 'spawnargs':{'attach':self.queue2}}

        child2 = base_consumer.ConsumerDesc(**pd2)

        child2_id = yield self.test_sup.spawn_child(child2)

        # Tell the first consumer to pass results to the second!
        #res = yield child1.set_params({'queues':[self.queue2]})

        yield self.test_sup.send(self.queue1, 'data', dmsg)

        yield pu.asleep(1)

        msg_cnt = yield child1.get_msg_count()
        received = msg_cnt.get('received', {})
        sent = msg_cnt.get('sent', {})
        self.assertEqual(sent.get(self.queue2), 2)
        self.assertEqual(received.get(self.queue1), 2)

        msg_cnt = yield child2.get_msg_count()
        received = msg_cnt.get('received', {})
        sent = msg_cnt.get('sent')
        self.assertEqual(sent, {})
        self.assertEqual(received.get(self.queue2), 1)

        yield child1.shutdown()
        yield child2.shutdown()
    def op_publish(self, content, headers, msg):
        """Service operation: Publish data message on a topic
        """
        logging.debug(self.__class__.__name__ +', op_'+ headers['op'] +' Received: ' +  str(headers))
        publication = dataobject.Resource.decode(content)
        logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication')
        #logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication: \n' + str(publication))
        
        #Get the data
        data = publication.data
        
        # Get the Topic
        topic_ref = publication.topic_ref
        topic = yield self.reg.get(topic_ref.reference(head=True))
        if not topic:
            logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', topic invalid!')
            yield self.reply_err(msg, 'Topic does not exist')
            return
        
        #Check publisher is valid!
        publisher = dm_resource_descriptions.PublisherResource()
        publisher.publisher = publication.publisher
        
        # Get the publications which this process is registered fro
        reg_pubs = yield self.reg.find(publisher, regex=False, attnames=['publisher'])
        
        valid = False
        for pub in reg_pubs:
            
            if topic.reference(head=True) in pub.topics:
                valid = True
                logging.debug(self.__class__.__name__ + '; Publishing to topic: \n' + str(topic))
                break
        
        if not valid:
            logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publisher not registered for topic!')
            yield self.reply_err(msg, 'Publisher not registered for topic!')
            return
        
        if not data.notification:
            data.notification = 'Data topic: ' + topic.name
        
        data.timestamp = pu.currenttime()
        
        # Todo: impersonate message as from sender -
        #@Todo - Move the actual publish command back to the client side!
        yield self.send(topic.queue.name, 'data', data.encode(), {})

        logging.info(self.__class__.__name__ + ': op_'+ headers['op'] + ' Success!')
        yield self.reply_ok(msg, '')
 def onschedule(self, queue='', max_points=50, **kwargs):
     '''
     This method is called when it is time to actually send the results
     in this case it is not needed, but must be over-riden...
     '''
         
     # Count the messages recieved
     total = 0
     for k,v in self.receive_cnt.items():
         total += v
     
     # Count the messages this interval
     interval = 0 
     for k,v in self.interval_cnt.items():
         interval += v
         
         
     notification = '''Message Counter has received %s messages, %s since last report''' \
                    % (total, interval)
         
     
     description = [('interval#','string', 'Interval Number'),
                 ('interval','number','Messages this Interval'),
                 ('total','number','Total Messages')]
     self.interval_number +=1
     
     self.data.append([str(self.interval_number),interval,total])
     
     dlen = len(self.data)
     if dlen > max_points:
         self.data = self.data[dlen-max_points : ]
     
     data_table = gviz_api.DataTable(description)
     data_table.LoadData(self.data)
     #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
     json = data_table.ToJSon()
         
     # Make message for the screen below
     msg = '<p>Number of messages received so far by Web Viz Consumer: %s </p>\n' % total
     msg += '<p>Timestamp: %s </p>\n' % pu.currenttime()
     
     page = line_template % {'msg':msg,'json':json}
         
     self.queue_result(queue,page,'Google Viz of message counts')
 def digest(self):
     
     logging.info(self.__class__.__name__ +"; Digesting results!")
     
     # Stop the loop if it is running - start again when next data is received
     if self.loop_running:
         #self.loop.stop()
         self.loop_running = False
     
     args = dict(self.params)
     args.update(self.deliver)
     
     yield defer.maybeDeferred(self.onschedule, **args)
     
     yield self.deliver_messages()
     # Update last_delivered
     self.interval_cnt={} # Reset the interval receive count
     self.last_delivered = pu.currenttime()
     logging.info(self.__class__.__name__ +"; digest: Finished sending results")
Beispiel #10
0
    def onschedule(self, queue='', max_points=50, **kwargs):
        '''
        This method is called when it is time to actually send the results
        in this case it is not needed, but must be over-riden...
        '''

        # Count the messages recieved
        total = 0
        for k, v in self.receive_cnt.items():
            total += v

        # Count the messages this interval
        interval = 0
        for k, v in self.interval_cnt.items():
            interval += v


        notification = '''Message Counter has received %s messages, %s since last report''' \
                       % (total, interval)

        description = [('interval#', 'string', 'Interval Number'),
                       ('interval', 'number', 'Messages this Interval'),
                       ('total', 'number', 'Total Messages')]
        self.interval_number += 1

        self.data.append([str(self.interval_number), interval, total])

        dlen = len(self.data)
        if dlen > max_points:
            self.data = self.data[dlen - max_points:]

        data_table = gviz_api.DataTable(description)
        data_table.LoadData(self.data)
        #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
        json = data_table.ToJSon()

        # Make message for the screen below
        msg = '<p>Number of messages received so far by Web Viz Consumer: %s </p>\n' % total
        msg += '<p>Timestamp: %s </p>\n' % pu.currenttime()

        page = line_template % {'msg': msg, 'json': json}

        self.queue_result(queue, page, 'Google Viz of message counts')
    def queue_result(self,queue, data=None, notification=''):
        
        if isinstance(data, DatasetType):
            msg = dap_tools.ds2dap_msg(data)
        elif isinstance(data, str):
            msg = StringMessageObject()
            msg.data=data
        elif isinstance(data, dict):
            msg = DictionaryMessageObject()
            msg.data=data
        elif not data:
            msg = DataMessageObject()
        else:
            raise RuntimeError('Invalid data type passed to send_result in class %s: type:' % (self.__class__.__name__, type(data)))

        if not notification:
            notification = self.__class__.__name__ + ' received a message!'

        msg.notification = notification

        msg.timestamp = pu.currenttime()
        
        self.msgs_to_send.append((queue, msg.encode()))
 def ondata(self, data, notification, timestamp, queue='', max_points=50, **kwargs):
     
     
     gname = data.keys()
     grid = data[gname[0]]
     for name in grid.keys():
         if not name == 'time':
             vname = name
     
     if not 'time' in grid.keys():
         raise RuntimeError('No axis time in grid dataset')
         
     if not len(grid.keys()) == 2:
         raise RuntimeError('Only time series are allowed in this consumer!')
         
     description = [('time','string', 'time (Seconds)'),
                 (vname,'number', vname)]
     
     vals = zip(grid['time'], grid[vname])
     for val in vals:
         self.pdata.append([str(val[0]),float(val[1])])
     
     dlen = len(self.pdata)
     if dlen > max_points:
         self.pdata = self.pdata[dlen-max_points : ]
     
     data_table = gviz_api.DataTable(description)
     data_table.LoadData(self.pdata)
     #json = data_table.ToJSon(columns_order=("name", "salary"),order_by="salary")
     json = data_table.ToJSon()
         
     # Make message for the screen below
     msg = '<p>Timestamp: %s </p>\n' % pu.currenttime()
     
     page = line_template % {'msg':msg,'json':json}
         
     self.queue_result(queue,page,'Google Viz of message counts')
    def op_data(self, content, headers, msg):

        logging.debug(self.__class__.__name__ +', MSG Received: ' + str(headers))

        logging.info(self.__class__.__name__ + '; Calling data process!')

        # Keep a record of messages received
        #@Note this could get big! What todo?
                
        self.receive_cnt[headers.get('receiver')] += 1
        #self.received_msg.append(content) # Do not keep the messages!

        # Unpack the message and turn it into data
        datamessage = dataobject.DataObject.decode(content)
        if isinstance(datamessage, DAPMessageObject):
            data = dap_tools.dap_msg2ds(datamessage)
        elif isinstance(datamessage, (StringMessageObject, DictionaryMessageObject)):
            data = datamessage.data
        else:
            data = None

        notification = datamessage.notification
        timestamp = datamessage.timestamp

        # Build the keyword args for ondata
        args = dict(self.params)
        args.update(self.deliver)

        logging.debug('**ARGS to ondata:'+str(args))
        yield defer.maybeDeferred(self.ondata, data, notification, timestamp, **args)

        logging.info(self.__class__.__name__ +"; op_data: Finished data processing")


        # Is this a consumer with digest delivery?
        if not self.delivery_interval:
            # if not send the messages from ondata...
            yield self.deliver_messages()
            logging.info(self.__class__.__name__ +"; op_data: Finished sending results")
            
        else: # Do the digets thing...
            
            if self.interval_cnt.has_key(headers.get('receiver')):
                self.interval_cnt[headers.get('receiver')] += 1
            else:
                self.interval_cnt[headers.get('receiver')] = 1
            
            
            logging.debug(self.__class__.__name__ +"; op_data: digest state: \n" + \
                          "Last Delivered: " +str(self.last_delivered) +";\n" +\
                          "Loop Running: " +str(self.loop_running))
            
            # First time data has arrived?
            if self.last_delivered == None:
                self.last_delivered = pu.currenttime()
                    
            if not self.loop_running:

                # Is it already time to go?
                if self.last_delivered + self.delivery_interval <= pu.currenttime():            
                    yield self.digest()
                        
                # if data has arrived but it is not yet time to deliver, schedule a call back
                else:
                    self.loop_running = True
                    delta_t = self.last_delivered + self.delivery_interval - pu.currenttime()
                    logging.debug('Scheduling a call back in %s seconds' % delta_t)
                    #self.loop.start(delta_t)
                    reactor.callLater(delta_t, self.digest)
    def test_send_chain(self):
        pd1={'name':'consumer_number_1',
                 'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
                 'procclass':'ForwardingConsumer',
                 'spawnargs':{'attach':self.queue1,\
                              'process parameters':{},\
                              'delivery queues':{'queues':[self.queue2]}}\
                    }
        child1 = base_consumer.ConsumerDesc(**pd1)

        child1_id = yield self.test_sup.spawn_child(child1)

        dmsg = DataMessageObject()
        dmsg.notifcation = 'Junk'
        dmsg.timestamp = pu.currenttime()
        dmsg = dmsg.encode()
        
        yield self.test_sup.send(self.queue1, 'data', dmsg)
        
        yield pu.asleep(1)
        
        msg_cnt = yield child1.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent.get(self.queue2),1)
        self.assertEqual(received.get(self.queue1),1)
        
        
        #Spawn another process to listen to queue 2  
        pd2={'name':'consumer_number_2', \
                 'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
                 'procclass':'ForwardingConsumer',\
                 'spawnargs':{'attach':self.queue2}}
        
        child2 = base_consumer.ConsumerDesc(**pd2)
                
        child2_id = yield self.test_sup.spawn_child(child2)
       
        # Tell the first consumer to pass results to the second!
        #res = yield child1.set_params({'queues':[self.queue2]})
        
        yield self.test_sup.send(self.queue1, 'data', dmsg)
        
        yield pu.asleep(1)

        msg_cnt = yield child1.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent.get(self.queue2),2)
        self.assertEqual(received.get(self.queue1),2)
        
        
        msg_cnt = yield child2.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent')
        self.assertEqual(sent,{})
        self.assertEqual(received.get(self.queue2),1)
        
        
        yield child1.shutdown()
        yield child2.shutdown()