예제 #1
0
    def setUp(self):
        yield self._start_container()
        #self.sup = yield self._spawn_processes(services)

        #Create two test queues
        queue1=dataobject.create_unique_identity()
        queue_properties = {queue1:{'name_type':'fanout', 'args':{'scope':'global'}}}
        yield bootstrap.declare_messaging(queue_properties)
        self.queue1 = queue1
예제 #2
0
    def setUp(self):
        yield self._start_container()
        #self.sup = yield self._spawn_processes(services)

        #Create two test queues
        queue1 = dataobject.create_unique_identity()
        queue_properties = {
            queue1: {
                'name_type': 'fanout',
                'args': {
                    'scope': 'global'
                }
            }
        }
        yield bootstrap.declare_messaging(queue_properties)
        self.queue1 = queue1

        queue2 = dataobject.create_unique_identity()
        queue_properties = {
            queue2: {
                'name_type': 'fanout',
                'args': {
                    'scope': 'global'
                }
            }
        }
        yield bootstrap.declare_messaging(queue_properties)
        self.queue2 = queue2

        queue3 = dataobject.create_unique_identity()
        queue_properties = {
            queue3: {
                'name_type': 'fanout',
                'args': {
                    'scope': 'global'
                }
            }
        }
        yield bootstrap.declare_messaging(queue_properties)
        self.queue3 = queue3
예제 #3
0
    def setUp(self):
        yield self._start_container()
        #self.sup = yield self._spawn_processes(services)

        #Create a test queue
        queue1 = dataobject.create_unique_identity()
        queue_properties = {
            queue1: {
                'name_type': 'fanout',
                'args': {
                    'scope': 'global'
                }
            }
        }
        yield bootstrap.declare_messaging(queue_properties)
        self.queue1 = queue1

        # Create a dataset to test with
        ds = dap_tools.simple_grid_dataset()
        fname = os.path.join(TEST_ARCHIVE_PATH, ds.name + '.nc')
        self.fname_test = fname

        fid = open(fname, "w")
        fid.close()

        # Create a persister process
        pd1 = {
            'name': 'persister_number_1',
            'module': 'ion.services.dm.preservation.persister',
            'procclass': 'PersisterConsumer',
            'spawnargs': {
                'attach': self.queue1,
                'process parameters': {
                    'filename': fname
                }
            }
        }

        self.child1 = base_consumer.ConsumerDesc(**pd1)
        child1_id = yield self.test_sup.spawn_child(self.child1)

        # Don't do this - you can only get the instance in a test case -
        # this is not a valid pattern in OTP
        self.dc1 = self._get_procinstance(child1_id)
        # Make sure it is up and working!
        self.assertIn(self.queue1, self.dc1.dataReceivers)

        self.ds = ds
        self.fname = fname
예제 #4
0
 def create_queue(self):
     """
     Create a queue
     @TODO fix the hack - This should come from the exchange registry!
     """
     
     queue = dm_resource_descriptions.Queue()
     queue.name = dataobject.create_unique_identity()
     queue.type = 'fanout'
     queue.args = {'scope':'global'}
     
     queue_properties = {queue.name:{'name_type':queue.type, 'args':queue.args}}
     # This should come from the COI Exchange registry
     yield bootstrap.declare_messaging(queue_properties)
     
     defer.returnValue(queue)
예제 #5
0
    def create_queue(self):
        """
        Create a queue
        @TODO fix the hack - This should come from the exchange registry!
        """

        queue = dm_resource_descriptions.Queue()
        queue.name = dataobject.create_unique_identity()
        queue.type = 'fanout'
        queue.args = {'scope': 'global'}

        queue_properties = {
            queue.name: {
                'name_type': queue.type,
                'args': queue.args
            }
        }
        # This should come from the COI Exchange registry
        yield bootstrap.declare_messaging(queue_properties)

        defer.returnValue(queue)
예제 #6
0
    def setUp(self):
        yield self._start_container()
        #self.sup = yield self._spawn_processes(services)

        #Create a test queue
        queue1=dataobject.create_unique_identity()
        queue_properties = {queue1:{'name_type':'fanout', 'args':{'scope':'global'}}}
        yield bootstrap.declare_messaging(queue_properties)
        self.queue1 = queue1
        
        
        # Create a dataset to test with
        ds=dap_tools.simple_grid_dataset()
        fname = os.path.join(TEST_ARCHIVE_PATH, ds.name +'.nc')
        self.fname_test = fname

        fid=open(fname, "w")
        fid.close()
        
        
        # Create a persister process
        pd1={'name':'persister_number_1',
                 'module':'ion.services.dm.preservation.persister',
                 'procclass':'PersisterConsumer',
                 'spawnargs':{'attach':self.queue1,
                              'process parameters':{'filename':fname}}}
        
        self.child1 = base_consumer.ConsumerDesc(**pd1)
        child1_id = yield self.test_sup.spawn_child(self.child1)
        
        # Don't do this - you can only get the instance in a test case -
        # this is not a valid pattern in OTP
        self.dc1 = self._get_procinstance(child1_id)
        # Make sure it is up and working!
        self.assertIn(self.queue1,self.dc1.dataReceivers)
        
        self.ds = ds
        self.fname = fname
예제 #7
0
    def test_exampleconsumer(self):
        '''
        @Brief Example Consumer is a demonstration of a more complex data consumer.
        It uses DAP data messages and provides qaqc and event results on two
        seperate queues.
        '''
        dpsc = DataPubsubClient(self.sup)
        
        #Create and register 3 topics!
        topic_raw = PubSubTopicResource.create("topic_raw","oceans, oil spill, fun things to do") 
        topic_raw = yield dpsc.define_topic(topic_raw)


        #Create and register self.sup as a publisher
        publisher = PublisherResource.create('Test Publisher', self.sup, topic_raw, 'DataObject')
        publisher = yield dpsc.define_publisher(publisher)

        logging.info('Defined Publisher: '+str(publisher))

        # === Create a Consumer and queues - this will become part of define_subscription.
        
        #Create two test queues - don't use topics to test the consumer
        # To be replaced when the subscription service is ready
        evt_queue=dataobject.create_unique_identity()
        queue_properties = {evt_queue:{'name_type':'fanout', 'args':{'scope':'global'}}}
        yield bootstrap.declare_messaging(queue_properties)

        pr_queue=dataobject.create_unique_identity()
        queue_properties = {pr_queue:{'name_type':'fanout', 'args':{'scope':'global'}}}
        yield bootstrap.declare_messaging(queue_properties)

        pd1={'name':'example_consumer_1',
                 'module':'ion.services.dm.distribution.consumers.example_consumer',
                 'procclass':'ExampleConsumer',
                 'spawnargs':{'attach':topic_raw.queue.name,\
                              'Process Parameters':{},\
                              'delivery queues':\
                              {'event_queue':evt_queue,\
                               'processed_queue':pr_queue}}\
                    }

        child1 = base_consumer.ConsumerDesc(**pd1)

        child1_id = yield self.test_sup.spawn_child(child1)


        pd2={'name':'example_consumer_2',
                 'module':'ion.services.dm.distribution.consumers.logging_consumer',
                 'procclass':'LoggingConsumer',
                 'spawnargs':{'attach':evt_queue,\
                              'Process Parameters':{}}\
                    }
        child2 = base_consumer.ConsumerDesc(**pd2)

        child2_id = yield self.test_sup.spawn_child(child2)

        pd3={'name':'example_consumer_3',
                 'module':'ion.services.dm.distribution.consumers.logging_consumer',
                 'procclass':'LoggingConsumer',
                 'spawnargs':{'attach':pr_queue,\
                              'Process Parameters':{}}\
                    }
        child3 = base_consumer.ConsumerDesc(**pd3)

        child3_id = yield self.test_sup.spawn_child(child3)

        # === End of stuff that will be replaced with Subscription method...


        # Create an example data message
        dmsg = dap_tools.simple_datamessage(\
            {'DataSet Name':'Simple Data','variables':\
                {'time':{'long_name':'Data and Time','units':'seconds'},\
                'height':{'long_name':'person height','units':'meters'}}}, \
            {'time':(101,102,103,104,105,106,107,108,109,110), \
            'height':(5,2,4,5,-1,9,3,888,3,4)})
        
        result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg)
        if result:
            logging.info('Published Message')
        else:
            logging.info('Failed to Published Message')


        # Need to await the delivery of data messages into the consumers
        yield pu.asleep(1)

        msg_cnt = yield child1.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent.get(evt_queue),2)
        self.assertEqual(sent.get(pr_queue),1)
        self.assertEqual(received.get(topic_raw.queue.name),1)
        
        msg_cnt = yield child2.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent,{})
        self.assertEqual(received.get(evt_queue),2)
        
        msg_cnt = yield child3.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent,{})
        self.assertEqual(received.get(pr_queue),1)
        

        # Publish a second message with different data
        dmsg = dap_tools.simple_datamessage(\
            {'DataSet Name':'Simple Data','variables':\
                {'time':{'long_name':'Data and Time','units':'seconds'},\
                'height':{'long_name':'person height','units':'meters'}}}, \
            {'time':(111,112,123,114,115,116,117,118,119,120), \
            'height':(8,986,4,-2,-1,5,3,1,4,5)})
        
        result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg)

        # Need to await the delivery of data messages into the consumers
        yield pu.asleep(1)

        msg_cnt = yield child1.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent.get(evt_queue),5)
        self.assertEqual(sent.get(pr_queue),2)
        self.assertEqual(received.get(topic_raw.queue.name),2)
        
        msg_cnt = yield child2.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent,{})
        self.assertEqual(received.get(evt_queue),5)
        
        msg_cnt = yield child3.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent,{})
        self.assertEqual(received.get(pr_queue),2)
예제 #8
0
    def test_pubsub(self):

        dpsc = DataPubsubClient(self.sup)
        
        # Create and Register a topic
        topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do")        
        topic = yield dpsc.define_topic(topic)
        logging.info('Defined Topic: '+str(topic))

        #Create and register self.sup as a publisher
        publisher = PublisherResource.create('Test Publisher', self.sup, topic, 'DataObject')
        publisher = yield dpsc.define_publisher(publisher)

        logging.info('Defined Publisher: '+str(publisher))
        

        
        # === Create a Consumer and queues - this will become part of define_subscription.
        
        #Create two test queues - don't use topics to test the consumer
        # To be replaced when the subscription service is ready
        queue1=dataobject.create_unique_identity()
        queue_properties = {queue1:{'name_type':'fanout', 'args':{'scope':'global'}}}
        yield bootstrap.declare_messaging(queue_properties)

        queue2=dataobject.create_unique_identity()
        queue_properties = {queue2:{'name_type':'fanout', 'args':{'scope':'global'}}}
        yield bootstrap.declare_messaging(queue_properties)

        pd1={'name':'example_consumer_1',
                 'module':'ion.services.dm.distribution.consumers.forwarding_consumer',
                 'procclass':'ForwardingConsumer',
                 'spawnargs':{'attach':topic.queue.name,\
                              'process parameters':{},\
                              'delivery queues':{'queues':[queue1,queue2]}}\
                    }
        child1 = base_consumer.ConsumerDesc(**pd1)

        child1_id = yield self.test_sup.spawn_child(child1)

        # === End to be replaces with Define_Consumer


        # Create and send a data message
        data = {'Data':'in a dictionary'}
        result = yield dpsc.publish(self.sup, topic.reference(), data)
        if result:
            logging.info('Published Message')
        else:
            logging.info('Failed to Published Message')

        # Need to await the delivery of data messages into the (separate) consumers
        yield pu.asleep(1)

        msg_cnt = yield child1.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent.get(queue1),1)
        self.assertEqual(sent.get(queue2),1)
        self.assertEqual(received.get(topic.queue.name),1)


        # === Create a Consumer - this will become part of define_subscription.
        
        pd2={'name':'example_consumer_2',
                 'module':'ion.services.dm.distribution.consumers.logging_consumer',
                 'procclass':'LoggingConsumer',
                 'spawnargs':{'attach':queue1,\
                              'process parameters':{},\
                              'delivery queues':{}}\
                    }
        child2 = base_consumer.ConsumerDesc(**pd2)

        child2_id = yield self.test_sup.spawn_child(child2)

        # === End of what will become part of the subscription definition

        # Send the simple message again
        result = yield dpsc.publish(self.sup, topic.reference(), data)
        
        # Need to await the delivery of data messages into the (separate) consumers
        yield pu.asleep(1)

        msg_cnt = yield child1.get_msg_count()

        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent.get(queue1),2)
        self.assertEqual(sent.get(queue2),2)
        self.assertEqual(received.get(topic.queue.name),2)
        
        msg_cnt = yield child2.get_msg_count()
        received = msg_cnt.get('received',{})
        sent = msg_cnt.get('sent',{})
        self.assertEqual(sent,{})
        self.assertEqual(received.get(queue1),1)