def test_sample(self): result = yield self.driver_client.initialize('some arg') dpsc = DataPubsubClient(self.sup) topicname = 'SBE49 Topic' topic = PubSubTopicResource.create(topicname,"") # Use the service to create a queue and register the topic topic = yield dpsc.define_topic(topic) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create(topicname,'') subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } subscription = yield dpsc.define_subscription(subscription) logging.info('Defined subscription: '+str(subscription)) params = {} params['publish-to'] = topic.RegistryIdentity yield self.driver_client.configure_driver(params) cmd1 = [['ds', 'now']] result = yield self.driver_client.execute(cmd1) self.assertEqual(result['status'], 'OK') yield pu.asleep(1) result = yield self.driver_client.disconnect(['some arg'])
def test_sample(self): result = yield self.driver_client.initialize('some arg') dpsc = DataPubsubClient(self.sup) topicname = 'SBE49 Topic' topic = PubSubTopicResource.create(topicname, "") # Use the service to create a queue and register the topic topic = yield dpsc.define_topic(topic) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create(topicname, '') subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } subscription = yield dpsc.define_subscription(subscription) logging.info('Defined subscription: ' + str(subscription)) params = {} params['publish-to'] = topic.RegistryIdentity yield self.driver_client.configure_driver(params) cmd1 = [['ds', 'now']] result = yield self.driver_client.execute(cmd1) self.assertEqual(result['status'], 'OK') yield pu.asleep(1) result = yield self.driver_client.disconnect(['some arg'])
def test_create_consumer_args1(self): ''' @Brief Test business logic to create consumer args from a workflow Make sure teh basics of the workflow processing work to create consumer args. ''' # Create and Register a topic topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do") # Use the service to create a queue and register the topic topic = yield self.pubsub.create_and_register_topic(topic) # A condition to search for the topic t_search = PubSubTopicResource.create('Davids Topic','') subscription = SubscriptionResource() subscription.topic1 = t_search subscription.workflow = {'consumer1':{'module':'path.to.module','consumerclass':'<ConsumerClassName>',\ 'attach':'topic1',\ 'process parameters':{'param1':'my parameter'}}} subscription = yield self.pubsub.create_consumer_args(subscription) logging.debug('Defined subscription consumers: '+str(subscription.consumer_args)) self.assertEqual({'consumer1': {'procclass': '<ConsumerClassName>', 'name': 'consumer1', 'module': 'path.to.module', 'spawnargs': { 'attach': [topic.queue.name], 'delivery interval': None, 'delivery queues': {}, 'process parameters': {'param1': 'my parameter'}}}}, subscription.consumer_args)
def test_create_consumer_args3(self): ''' @Brief Test business logic to create consumer args from a workflow In this test two consumers attach to the same queue created by the workflow ''' # Create and Register a topic topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do") # Use the service to create a queue and register the topic topic = yield self.pubsub.create_and_register_topic(topic) # A condition to search for the topic t_search = PubSubTopicResource.create('Davids Topic','') subscription = SubscriptionResource() subscription.topic1 = t_search subscription.workflow = { 'consumer1':{'module':'path.to.module','consumerclass':'<ConsumerClassName>',\ 'attach':'topic1',\ 'process parameters':{'param1':'my parameter1'}}, 'consumer2':{'module':'path.to.module2','consumerclass':'<ConsumerClassName2>',\ 'attach':[['consumer1','deliver_to']],\ 'Process Parameters':{'param2':'my parameter2'}}, 'consumer3':{'module':'path.to.module3','consumerclass':'<ConsumerClassName3>',\ 'attach':[['consumer1','deliver_to'],'topic1']} } subscription = yield self.pubsub.create_consumer_args(subscription) logging.debug('Defined subscription consumers: '+str(subscription.consumer_args)) consume2_attach = subscription.consumer_args['consumer2']['spawnargs']['attach'] self.assertEqual({'module': 'path.to.module', 'name': 'consumer1', 'procclass': '<ConsumerClassName>', 'spawnargs': {'attach': [topic.queue.name], 'delivery interval': None, 'delivery queues': {'deliver_to': consume2_attach[0]}, 'process parameters': {'param1': 'my parameter1'}}}, subscription.consumer_args['consumer1']) self.assertEqual({'procclass': '<ConsumerClassName3>', 'name': 'consumer3', 'module': 'path.to.module3', 'spawnargs': { 'attach': [consume2_attach[0], topic.queue.name], 'delivery interval': None, 'delivery queues': {}, 'process parameters': {}}}, subscription.consumer_args['consumer3'])
def test_create_subscription2(self): ''' @Brief Create a subscription! ''' # Create and Register a topic topic1 = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do") # Use the service to create a queue and register the topic topic1 = yield self.pubsub.create_and_register_topic(topic1) topic2 = PubSubTopicResource.create('Johns Topic',"oceans, mbari, working really hard") # Use the service to create a queue and register the topic topic2 = yield self.pubsub.create_and_register_topic(topic2) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create('Davids Topic','') subscription.topic2 = PubSubTopicResource.create('','oceans') subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'consumerclass':'ForwardingConsumer',\ 'attach':'topic1'}, 'consumer2': {'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'consumerclass':'ForwardingConsumer',\ 'attach':[['consumer1','queues']]} } subscription = yield self.pubsub.create_subscription(subscription) logging.info('Defined subscription: '+str(subscription)) msg=DataMessageObject() self.sup.send(topic1.queue.name,'data',msg.encode()) # Wait for message to be received yield pu.asleep(1) child1_id = self.pubsub.get_child_id('consumer1') dc1 = self._get_procinstance(child1_id) rec = dc1.receive_cnt[topic1.queue.name] self.assertEqual(rec,1) child2_id = self.pubsub.get_child_id('consumer2') dc2 = self._get_procinstance(child2_id) q = subscription.consumer_args['consumer2']['spawnargs']['attach'] rec = dc2.receive_cnt[q[0]] self.assertEqual(rec,1)
def create_producers(proc,n=1): dpsc = pubsub_service.DataPubsubClient(proc=proc) for i in range(n): tname = 'topic name '+str(i) ka = 'keyword a' kb = 'keyword b' if (i/2)*2 == i: keyword = ka else: keyword = kb topic = PubSubTopicResource.create(tname,keyword) topic = yield dpsc.define_topic(topic) dspname = 'data_stream_producer_'+str(i) interval = random.randint(1,10) dsp={'name':dspname, 'module':'ion.services.dm.util.data_stream_producer', 'procclass':'DataStreamProducer', 'spawnargs':{'delivery queue':topic.queue.name, 'delivery interval':interval}} child = ProcessDesc(**dsp) child_id = yield proc.spawn_child(child)
def create_producers(proc, n=1): dpsc = pubsub_service.DataPubsubClient(proc=proc) for i in range(n): tname = 'topic name ' + str(i) ka = 'keyword a' kb = 'keyword b' if (i / 2) * 2 == i: keyword = ka else: keyword = kb topic = PubSubTopicResource.create(tname, keyword) topic = yield dpsc.define_topic(topic) dspname = 'data_stream_producer_' + str(i) interval = random.randint(1, 10) dsp = { 'name': dspname, 'module': 'ion.services.dm.util.data_stream_producer', 'procclass': 'DataStreamProducer', 'spawnargs': { 'delivery queue': topic.queue.name, 'delivery interval': interval } } child = ProcessDesc(**dsp) child_id = yield proc.spawn_child(child)
def test_create_topic(self): # Create and Register a topic topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do") # Make sure the topic starts out with blank stuff here... self.assertEqual(topic.RegistryBranch,'master') self.assertEqual(topic.RegistryIdentity,'') self.assertEqual(topic.queue.type,'') # Use the service to create a queue and register the topic topic = yield self.pubsub.create_and_register_topic(topic) # Make sure the queue properties were set. self.assertEqual(topic.queue.type,'fanout') #Spawn an baseconsumer and make sure a message is received on the new queue pd1={'name':'consumer_number_1', 'module':'ion.services.dm.distribution.consumers.logging_consumer', 'procclass':'LoggingConsumer', 'spawnargs':{'attach':[topic.queue.name]}} child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) msg=DataMessageObject() self.sup.send(topic.queue.name,'data',msg.encode()) yield pu.asleep(1) dc1 = self._get_procinstance(child1_id) #print 'DC1',dc1.receive_cnt rec = dc1.receive_cnt[topic.queue.name] self.assertEqual(rec,1)
def op_start_instrument_agent(self, content, headers, msg): """ Service operation: Starts an instrument agent for a type of instrument. """ if 'instrumentID' in content: inst_id = str(content['instrumentID']) else: raise ValueError("Input for instrumentID not present") if 'model' in content: model = str(content['model']) else: raise ValueError("Input for model not present") if model != 'SBE49': raise ValueError("Only SBE49 supported!") agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if agent_pid: raise StandardError("Agent already started for instrument " + str(inst_id)) simulator = Simulator(inst_id) simulator.start() topicname = "Inst/RAW/" + inst_id topic = PubSubTopicResource.create(topicname, "") # Use the service to create a queue and register the topic topic = yield self.dpsc.define_topic(topic) iagent_args = {} iagent_args['instrument-id'] = inst_id driver_args = {} driver_args['port'] = simulator.port driver_args['publish-to'] = topic.RegistryIdentity iagent_args['driver-args'] = driver_args iapd = ProcessDesc( **{ 'name': 'SBE49IA', 'module': 'ion.agents.instrumentagents.SBE49_IA', 'class': 'SBE49InstrumentAgent', 'spawnargs': iagent_args }) iagent_id = yield self.spawn_child(iapd) iaclient = InstrumentAgentClient(proc=self, target=iagent_id) yield iaclient.register_resource(inst_id) yield self.reply_ok(msg, "OK")
def start(): """ Main function of bootstrap. Starts DM pubsub... """ logging.info("ION DM PubSub bootstrapping now...") startsvcs = [] #startsvcs.extend(dm_services) sup = yield bootstrap.bootstrap(ion_messaging, startsvcs) print 'STARTSVCS',startsvcs print 'ION_MESSAGING',ion_messaging print 'CONT_ARGS',ioninit.cont_args #yield create_producer(sup) dpsc = pubsub_service.DataPubsubClient(proc=sup) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create('Inst/RAW','') # Use the example consumer to create events... graph the number of events ''' subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } ''' subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.instrument_timeseries_consumer', 'consumerclass':'InstrumentTimeseriesConsumer',\ 'attach':'topic1'}, 'consumer3': {'module':'ion.services.dm.presentation.web_viz_consumer', 'consumerclass':'WebVizConsumer',\ 'attach':[['consumer1','queue']], 'process parameters':{'port':8180} } } subscription = yield dpsc.define_subscription(subscription) linfo = '\n================================================\n' linfo+= 'Open your web browser and look at: http://127.0.0.1:8180/ \n' linfo+= '================================================\n' logging.info(linfo)
def start(): """ Main function of bootstrap. Starts DM pubsub... """ logging.info("ION DM PubSub bootstrapping now...") startsvcs = [] #startsvcs.extend(dm_services) sup = yield bootstrap.bootstrap(ion_messaging, startsvcs) print 'STARTSVCS', startsvcs print 'ION_MESSAGING', ion_messaging print 'CONT_ARGS', ioninit.cont_args #yield create_producer(sup) dpsc = pubsub_service.DataPubsubClient(proc=sup) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create('Inst/RAW', '') # Use the example consumer to create events... graph the number of events ''' subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } ''' subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.instrument_timeseries_consumer', 'consumerclass':'InstrumentTimeseriesConsumer',\ 'attach':'topic1'}, 'consumer3': {'module':'ion.services.dm.presentation.web_viz_consumer', 'consumerclass':'WebVizConsumer',\ 'attach':[['consumer1','queue']], 'process parameters':{'port':8180} } } subscription = yield dpsc.define_subscription(subscription) linfo = '\n================================================\n' linfo += 'Open your web browser and look at: http://127.0.0.1:8180/ \n' linfo += '================================================\n' logging.info(linfo)
def test_create_topic(self): #dpsc = DataPubsubClient(self.pubsubSuper) dpsc = DataPubsubClient(self.sup) # Create and Register a topic """ DHE: not sure the driver should be creating the topic; for right now I'll have the test case do it. """ self.topic = PubSubTopicResource.create('SBE49 Topic', "oceans, oil spill") self.topic = yield dpsc.define_topic(self.topic) print 'TADA!'
def test_create_topic(self): #dpsc = DataPubsubClient(self.pubsubSuper) dpsc = DataPubsubClient(self.sup) # Create and Register a topic """ DHE: not sure the driver should be creating the topic; for right now I'll have the test case do it. """ self.topic = PubSubTopicResource.create('SBE49 Topic',"oceans, oil spill") self.topic = yield dpsc.define_topic(self.topic) print 'TADA!'
def test_create_subscription1(self): ''' @Brief Create a subscription! ''' # Create and Register a topic topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do") # Use the service to create a queue and register the topic topic = yield self.pubsub.create_and_register_topic(topic) # A condition to search for the topic t_search = PubSubTopicResource.create('Davids Topic','') subscription = SubscriptionResource() subscription.topic1 = t_search subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'consumerclass':'ForwardingConsumer',\ 'attach':'topic1'}} subscription = yield self.pubsub.create_subscription(subscription) logging.info('Defined subscription: '+str(subscription)) msg=DataMessageObject() self.sup.send(topic.queue.name,'data',msg.encode()) # Wait for message to be received yield pu.asleep(1) child_id = self.pubsub.get_child_id('consumer1') dc1 = self._get_procinstance(child_id) rec = dc1.receive_cnt[topic.queue.name] self.assertEqual(rec,1)
def op_start_instrument_agent(self, content, headers, msg): """ Service operation: Starts an instrument agent for a type of instrument. """ if 'instrumentID' in content: inst_id = str(content['instrumentID']) else: raise ValueError("Input for instrumentID not present") if 'model' in content: model = str(content['model']) else: raise ValueError("Input for model not present") if model != 'SBE49': raise ValueError("Only SBE49 supported!") agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if agent_pid: raise StandardError("Agent already started for instrument "+str(inst_id)) simulator = Simulator(inst_id) simulator.start() topicname = "Inst/RAW/"+inst_id topic = PubSubTopicResource.create(topicname,"") # Use the service to create a queue and register the topic topic = yield self.dpsc.define_topic(topic) iagent_args = {} iagent_args['instrument-id'] = inst_id driver_args = {} driver_args['port'] = simulator.port driver_args['publish-to'] = topic.RegistryIdentity iagent_args['driver-args'] = driver_args iapd = ProcessDesc(**{'name':'SBE49IA', 'module':'ion.agents.instrumentagents.SBE49_IA', 'class':'SBE49InstrumentAgent', 'spawnargs':iagent_args}) iagent_id = yield self.spawn_child(iapd) iaclient = InstrumentAgentClient(proc=self, target=iagent_id) yield iaclient.register_resource(inst_id) yield self.reply_ok(msg, "OK")
def create_producer(proc): dpsc = pubsub_service.DataPubsubClient(proc=proc) topic = PubSubTopicResource.create('time series','dap grid, timeseries, data') topic = yield dpsc.define_topic(topic) dspname = 'data_timeseries_stream_producer' dsp={'name':dspname, 'module':'ion.services.dm.util.dap_grid_timeseries_producer', 'procclass':'DapGridDataStreamProducer', 'spawnargs':{'delivery queue':topic.queue.name, 'delivery interval':5} } child = ProcessDesc(**dsp) child_id = yield proc.spawn_child(child)
def create_producer(proc): dpsc = pubsub_service.DataPubsubClient(proc=proc) topic = PubSubTopicResource.create('time series', 'dap grid, timeseries, data') topic = yield dpsc.define_topic(topic) dspname = 'data_timeseries_stream_producer' dsp = { 'name': dspname, 'module': 'ion.services.dm.util.dap_grid_timeseries_producer', 'procclass': 'DapGridDataStreamProducer', 'spawnargs': { 'delivery queue': topic.queue.name, 'delivery interval': 5 } } child = ProcessDesc(**dsp) child_id = yield proc.spawn_child(child)
def test_execute(self): """ Test the execute command to the Instrument Driver """ result = yield self.driver_client.initialize('some arg') dpsc = DataPubsubClient(self.sup) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create('SBE49 Topic','') #subscription.topic2 = PubSubTopicResource.create('','oceans') subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } subscription = yield dpsc.define_subscription(subscription) logging.info('Defined subscription: '+str(subscription)) cmd1 = [['ds', 'now']] #cmd1 = [['start', 'now']] #cmd2 = [['stop', 'now']] #cmd2 = [['pumpoff', '3600', '1']] result = yield self.driver_client.execute(cmd1) self.assertEqual(result['status'], 'OK') # DHE: wait a while... yield pu.asleep(1) #result = yield self.driver_client.execute(cmd2) #self.assertEqual(result['status'], 'OK') # DHE: disconnecting; a connect would probably be good. result = yield self.driver_client.disconnect(['some arg'])
def test_execute(self): """ Test the execute command to the Instrument Driver """ result = yield self.driver_client.initialize('some arg') dpsc = DataPubsubClient(self.sup) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create('SBE49 Topic', '') #subscription.topic2 = PubSubTopicResource.create('','oceans') subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } subscription = yield dpsc.define_subscription(subscription) logging.info('Defined subscription: ' + str(subscription)) cmd1 = [['ds', 'now']] #cmd1 = [['start', 'now']] #cmd2 = [['stop', 'now']] #cmd2 = [['pumpoff', '3600', '1']] result = yield self.driver_client.execute(cmd1) self.assertEqual(result['status'], 'OK') # DHE: wait a while... yield pu.asleep(1) #result = yield self.driver_client.execute(cmd2) #self.assertEqual(result['status'], 'OK') # DHE: disconnecting; a connect would probably be good. result = yield self.driver_client.disconnect(['some arg'])
def test_pubsub(self): dpsc = DataPubsubClient(self.sup) # Create and Register a topic topic = PubSubTopicResource.create('Davids Topic',"oceans, oil spill, fun things to do") topic = yield dpsc.define_topic(topic) logging.info('Defined Topic: '+str(topic)) #Create and register self.sup as a publisher publisher = PublisherResource.create('Test Publisher', self.sup, topic, 'DataObject') publisher = yield dpsc.define_publisher(publisher) logging.info('Defined Publisher: '+str(publisher)) # === Create a Consumer and queues - this will become part of define_subscription. #Create two test queues - don't use topics to test the consumer # To be replaced when the subscription service is ready queue1=dataobject.create_unique_identity() queue_properties = {queue1:{'name_type':'fanout', 'args':{'scope':'global'}}} yield bootstrap.declare_messaging(queue_properties) queue2=dataobject.create_unique_identity() queue_properties = {queue2:{'name_type':'fanout', 'args':{'scope':'global'}}} yield bootstrap.declare_messaging(queue_properties) pd1={'name':'example_consumer_1', 'module':'ion.services.dm.distribution.consumers.forwarding_consumer', 'procclass':'ForwardingConsumer', 'spawnargs':{'attach':topic.queue.name,\ 'process parameters':{},\ 'delivery queues':{'queues':[queue1,queue2]}}\ } child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) # === End to be replaces with Define_Consumer # Create and send a data message data = {'Data':'in a dictionary'} result = yield dpsc.publish(self.sup, topic.reference(), data) if result: logging.info('Published Message') else: logging.info('Failed to Published Message') # Need to await the delivery of data messages into the (separate) consumers yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(queue1),1) self.assertEqual(sent.get(queue2),1) self.assertEqual(received.get(topic.queue.name),1) # === Create a Consumer - this will become part of define_subscription. pd2={'name':'example_consumer_2', 'module':'ion.services.dm.distribution.consumers.logging_consumer', 'procclass':'LoggingConsumer', 'spawnargs':{'attach':queue1,\ 'process parameters':{},\ 'delivery queues':{}}\ } child2 = base_consumer.ConsumerDesc(**pd2) child2_id = yield self.test_sup.spawn_child(child2) # === End of what will become part of the subscription definition # Send the simple message again result = yield dpsc.publish(self.sup, topic.reference(), data) # Need to await the delivery of data messages into the (separate) consumers yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(queue1),2) self.assertEqual(sent.get(queue2),2) self.assertEqual(received.get(topic.queue.name),2) msg_cnt = yield child2.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(queue1),1)
def op_create_ingestion_datastream(self, content, headers, msg): """Service operation: declare new named datastream for ingestion 0) Decode content to Igestion Stream Resource 1) create inbound topic 2) create ingested topic 3) Register new data 3) start preservation of ingested 4) start preservation of inbound 5) start ingestion workflow Register progress and set lcstate along the way return the ingestiondatastream resource """ logging.debug(self.__class__.__name__ +', op_'+ headers['op'] +' Received: ' + str(headers)) isr = dataobject.DataObject.decode(content) logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', Ingestion Stream: \n' + str(publisher)) # Register the intended feed... isr = yield self.reg.define_ingestion_stream(isr) inbnd = irs.name + '.inbound' topic = PubSubTopicResource.create(name=inbnd,keywords='input') # Place holder - what to put, NOT RAW! topic = yield self.pubsub.define_topic(topic) isr.input_topic = topic.reference(heat=True) ingested = irs.name + '.ingested' topic = PubSubTopicResource.create(name=ingested,keywords='ingested') topic = yield self.pubsub.define_topic(topic) isr.ingested_topic = topic.reference(heat=True) # Register the feed topics... isr = yield self.reg.define_ingestion_stream(isr) #Create a DM Data Resource for this new feed. dmdr = DMDataResource.create_new_resource() dmdr.input_topic = isr.input_topic dmdr.ingested_topic = isr.ingested_topic # Register the data dmdr = yield self.datareg.define_data(dmdr) #Call the preservation service for the input stream arc = ArchiveResource.create_new_resource() arc.dmdataresource = dmdr.reference(head=true) arc.topic = isr.input_topic yield self.preserv.create_archive(arc) yield self.preserv.activate_persister # Register the data dmdr.input_archive = arc.reference(head=True) dmdr = yield self.datareg.define_data(dmdr) #Call the preservation service for the ingested stream arc = ArchiveResource.create_new_resource() arc.dmdataresource = dmdr.reference(head=true) arc.topic = isr.ingested_topic yield self.preserv.create_archive(arc) yield self.preserv.activate_persister # Register the data dmdr.ingested_archive = arc.reference(head=True) dmdr = yield self.datareg.define_data(dmdr) # Register the persisters isr.persisting_inout = True isr.persisting_ingested = True isr = yield self.reg.define_ingestion_stream(isr) #activate ingestion isr = yield self._activate_ingestion(isr, dmdr) isr = yield self.reg.define_ingestion_stream(isr) yield self.reply_ok(isr.encode())
def start(): """ Main function of bootstrap. Starts DM pubsub... """ logging.info("ION DM PubSub bootstrapping now...") startsvcs = [] startsvcs.extend(dm_services) sup = yield bootstrap.bootstrap(ion_messaging, startsvcs) print 'STARTSVCS', startsvcs print 'ION_MESSAGING', ion_messaging print 'CONT_ARGS', ioninit.cont_args nproducers = int(ioninit.cont_args.get('nproducers', 5)) print 'NPRODUCERS', nproducers yield create_producers(sup, nproducers) dpsc = pubsub_service.DataPubsubClient(proc=sup) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create('topic', '') """ # Use the example consumer to create events... log the events subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.example_consumer', 'consumerclass':'ExampleConsumer',\ 'attach':'topic1'}, 'consumer2': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':[['consumer1','event_queue']]} } # Log all the messages created subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } """ # Use the example consumer to create events... graph the number of events subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.example_consumer', 'consumerclass':'ExampleConsumer',\ 'attach':'topic1'}, 'consumer2': {'module':'ion.services.dm.distribution.consumers.message_count_consumer', 'consumerclass':'MessageCountConsumer',\ 'attach':[['consumer1','event_queue']],\ 'delivery interval':5, 'process parameters':{'max_points':5} }, 'consumer3': {'module':'ion.services.dm.presentation.web_viz_consumer', 'consumerclass':'WebVizConsumer',\ 'attach':[['consumer2','queue']], 'process parameters':{'port':8180} } } subscription = yield dpsc.define_subscription(subscription) linfo = '\n================================================\n' linfo += 'Open your web browser and look at: http://127.0.0.1:8180/ \n' linfo += '================================================\n' logging.info(linfo)
def test_exampleconsumer(self): ''' @Brief Example Consumer is a demonstration of a more complex data consumer. It uses DAP data messages and provides qaqc and event results on two seperate queues. ''' dpsc = DataPubsubClient(self.sup) #Create and register 3 topics! topic_raw = PubSubTopicResource.create("topic_raw","oceans, oil spill, fun things to do") topic_raw = yield dpsc.define_topic(topic_raw) #Create and register self.sup as a publisher publisher = PublisherResource.create('Test Publisher', self.sup, topic_raw, 'DataObject') publisher = yield dpsc.define_publisher(publisher) logging.info('Defined Publisher: '+str(publisher)) # === Create a Consumer and queues - this will become part of define_subscription. #Create two test queues - don't use topics to test the consumer # To be replaced when the subscription service is ready evt_queue=dataobject.create_unique_identity() queue_properties = {evt_queue:{'name_type':'fanout', 'args':{'scope':'global'}}} yield bootstrap.declare_messaging(queue_properties) pr_queue=dataobject.create_unique_identity() queue_properties = {pr_queue:{'name_type':'fanout', 'args':{'scope':'global'}}} yield bootstrap.declare_messaging(queue_properties) pd1={'name':'example_consumer_1', 'module':'ion.services.dm.distribution.consumers.example_consumer', 'procclass':'ExampleConsumer', 'spawnargs':{'attach':topic_raw.queue.name,\ 'Process Parameters':{},\ 'delivery queues':\ {'event_queue':evt_queue,\ 'processed_queue':pr_queue}}\ } child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) pd2={'name':'example_consumer_2', 'module':'ion.services.dm.distribution.consumers.logging_consumer', 'procclass':'LoggingConsumer', 'spawnargs':{'attach':evt_queue,\ 'Process Parameters':{}}\ } child2 = base_consumer.ConsumerDesc(**pd2) child2_id = yield self.test_sup.spawn_child(child2) pd3={'name':'example_consumer_3', 'module':'ion.services.dm.distribution.consumers.logging_consumer', 'procclass':'LoggingConsumer', 'spawnargs':{'attach':pr_queue,\ 'Process Parameters':{}}\ } child3 = base_consumer.ConsumerDesc(**pd3) child3_id = yield self.test_sup.spawn_child(child3) # === End of stuff that will be replaced with Subscription method... # Create an example data message dmsg = dap_tools.simple_datamessage(\ {'DataSet Name':'Simple Data','variables':\ {'time':{'long_name':'Data and Time','units':'seconds'},\ 'height':{'long_name':'person height','units':'meters'}}}, \ {'time':(101,102,103,104,105,106,107,108,109,110), \ 'height':(5,2,4,5,-1,9,3,888,3,4)}) result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg) if result: logging.info('Published Message') else: logging.info('Failed to Published Message') # Need to await the delivery of data messages into the consumers yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(evt_queue),2) self.assertEqual(sent.get(pr_queue),1) self.assertEqual(received.get(topic_raw.queue.name),1) msg_cnt = yield child2.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(evt_queue),2) msg_cnt = yield child3.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(pr_queue),1) # Publish a second message with different data dmsg = dap_tools.simple_datamessage(\ {'DataSet Name':'Simple Data','variables':\ {'time':{'long_name':'Data and Time','units':'seconds'},\ 'height':{'long_name':'person height','units':'meters'}}}, \ {'time':(111,112,123,114,115,116,117,118,119,120), \ 'height':(8,986,4,-2,-1,5,3,1,4,5)}) result = yield dpsc.publish(self.sup, topic_raw.reference(), dmsg) # Need to await the delivery of data messages into the consumers yield pu.asleep(1) msg_cnt = yield child1.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent.get(evt_queue),5) self.assertEqual(sent.get(pr_queue),2) self.assertEqual(received.get(topic_raw.queue.name),2) msg_cnt = yield child2.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(evt_queue),5) msg_cnt = yield child3.get_msg_count() received = msg_cnt.get('received',{}) sent = msg_cnt.get('sent',{}) self.assertEqual(sent,{}) self.assertEqual(received.get(pr_queue),2)
def start(): """ Main function of bootstrap. Starts DM pubsub... """ logging.info("ION DM PubSub bootstrapping now...") startsvcs = [] startsvcs.extend(dm_services) sup = yield bootstrap.bootstrap(ion_messaging, startsvcs) print 'STARTSVCS',startsvcs print 'ION_MESSAGING',ion_messaging print 'CONT_ARGS',ioninit.cont_args nproducers = int(ioninit.cont_args.get('nproducers',5)) print 'NPRODUCERS',nproducers yield create_producers(sup, nproducers) dpsc = pubsub_service.DataPubsubClient(proc=sup) subscription = SubscriptionResource() subscription.topic1 = PubSubTopicResource.create('topic','') """ # Use the example consumer to create events... log the events subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.example_consumer', 'consumerclass':'ExampleConsumer',\ 'attach':'topic1'}, 'consumer2': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':[['consumer1','event_queue']]} } # Log all the messages created subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.logging_consumer', 'consumerclass':'LoggingConsumer',\ 'attach':'topic1'} } """ # Use the example consumer to create events... graph the number of events subscription.workflow = { 'consumer1': {'module':'ion.services.dm.distribution.consumers.example_consumer', 'consumerclass':'ExampleConsumer',\ 'attach':'topic1'}, 'consumer2': {'module':'ion.services.dm.distribution.consumers.message_count_consumer', 'consumerclass':'MessageCountConsumer',\ 'attach':[['consumer1','event_queue']],\ 'delivery interval':5, 'process parameters':{'max_points':5} }, 'consumer3': {'module':'ion.services.dm.presentation.web_viz_consumer', 'consumerclass':'WebVizConsumer',\ 'attach':[['consumer2','queue']], 'process parameters':{'port':8180} } } subscription = yield dpsc.define_subscription(subscription) linfo = '\n================================================\n' linfo+= 'Open your web browser and look at: http://127.0.0.1:8180/ \n' linfo+= '================================================\n' logging.info(linfo)
def op_create_ingestion_datastream(self, content, headers, msg): """Service operation: declare new named datastream for ingestion 0) Decode content to Igestion Stream Resource 1) create inbound topic 2) create ingested topic 3) Register new data 3) start preservation of ingested 4) start preservation of inbound 5) start ingestion workflow Register progress and set lcstate along the way return the ingestiondatastream resource """ logging.debug(self.__class__.__name__ + ", op_" + headers["op"] + " Received: " + str(headers)) isr = dataobject.DataObject.decode(content) logging.info( self.__class__.__name__ + " recieved: op_" + headers["op"] + ", Ingestion Stream: \n" + str(publisher) ) # Register the intended feed... isr = yield self.reg.define_ingestion_stream(isr) inbnd = irs.name + ".inbound" topic = PubSubTopicResource.create(name=inbnd, keywords="input") # Place holder - what to put, NOT RAW! topic = yield self.pubsub.define_topic(topic) isr.input_topic = topic.reference(heat=True) ingested = irs.name + ".ingested" topic = PubSubTopicResource.create(name=ingested, keywords="ingested") topic = yield self.pubsub.define_topic(topic) isr.ingested_topic = topic.reference(heat=True) # Register the feed topics... isr = yield self.reg.define_ingestion_stream(isr) # Create a DM Data Resource for this new feed. dmdr = DMDataResource.create_new_resource() dmdr.input_topic = isr.input_topic dmdr.ingested_topic = isr.ingested_topic # Register the data dmdr = yield self.datareg.define_data(dmdr) # Call the preservation service for the input stream arc = ArchiveResource.create_new_resource() arc.dmdataresource = dmdr.reference(head=true) arc.topic = isr.input_topic yield self.preserv.create_archive(arc) yield self.preserv.activate_persister # Register the data dmdr.input_archive = arc.reference(head=True) dmdr = yield self.datareg.define_data(dmdr) # Call the preservation service for the ingested stream arc = ArchiveResource.create_new_resource() arc.dmdataresource = dmdr.reference(head=true) arc.topic = isr.ingested_topic yield self.preserv.create_archive(arc) yield self.preserv.activate_persister # Register the data dmdr.ingested_archive = arc.reference(head=True) dmdr = yield self.datareg.define_data(dmdr) # Register the persisters isr.persisting_inout = True isr.persisting_ingested = True isr = yield self.reg.define_ingestion_stream(isr) # activate ingestion isr = yield self._activate_ingestion(isr, dmdr) isr = yield self.reg.define_ingestion_stream(isr) yield self.reply_ok(isr.encode())