def test_stream_pub_sub(self): self.verified = Event() self.route = StreamRoute(exchange_point='xp_test', routing_key='route') def verify(message, route, stream): self.assertEquals(message, 'test') self.assertEquals(route, self.route) self.assertEquals(stream, '') self.verified.set() sub_proc = SimpleProcess() sub_proc.container = self.container sub1 = StreamSubscriber(process=sub_proc, exchange_name='sub1', callback=verify) sub1.start() self.queue_cleanup.append('sub1') pub_proc = SimpleProcess() pub_proc.container = self.container pub1 = StreamPublisher(process=pub_proc, stream_route=self.route) sub1.xn.bind(self.route.routing_key, pub1.xp) pub1.publish('test') self.assertTrue(self.verified.wait(2))
def _construct_stream_and_publisher(self, stream_name, stream_config): if log.isEnabledFor(logging.TRACE): # pragma: no cover log.trace("%r: _construct_stream_and_publisher: " "stream_name:%r, stream_config:\n%s", self._platform_id, stream_name, self._pp.pformat(stream_config)) decoder = IonObjectDeserializer(obj_registry=get_obj_registry()) if 'stream_def_dict' not in stream_config: # should not happen: PlatformAgent._validate_configuration validates this. log.error("'stream_def_dict' key not in configuration for stream %r" % stream_name) return stream_def_dict = stream_config['stream_def_dict'] stream_def_dict['type_'] = 'StreamDefinition' stream_def_obj = decoder.deserialize(stream_def_dict) self._stream_defs[stream_name] = stream_def_obj routing_key = stream_config['routing_key'] stream_id = stream_config['stream_id'] exchange_point = stream_config['exchange_point'] parameter_dictionary = stream_def_dict['parameter_dictionary'] log.debug("%r: got parameter_dictionary from stream_def_dict", self._platform_id) self._data_streams[stream_name] = stream_id self._param_dicts[stream_name] = ParameterDictionary.load(parameter_dictionary) stream_route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key) publisher = self._create_publisher(stream_id, stream_route) self._data_publishers[stream_name] = publisher log.debug("%r: created publisher for stream_name=%r", self._platform_id, stream_name)
def test_stream_pub_sub(self): self.verified = Event() self.route = StreamRoute(routing_key='stream_name') def verify(message, route, stream): self.assertEquals(message, 'test') self.assertEquals(route.routing_key, self.route.routing_key) self.assertTrue(route.exchange_point.startswith(get_sys_name())) self.assertEquals(stream, 'stream_name') self.verified.set() sub_proc = SimpleProcess() sub_proc.container = self.container sub1 = StreamSubscriber(process=sub_proc, exchange_name='stream_name', callback=verify) sub1.add_stream_subscription("stream_name") sub1.start() self.queue_cleanup.append('data.stream_name') pub_proc = SimpleProcess() pub_proc.container = self.container pub1 = StreamPublisher(process=pub_proc, stream=self.route) sub1.xn.bind(self.route.routing_key, pub1.xp) pub1.publish('test') self.assertTrue(self.verified.wait(2))
def __init__(self, process, stream, **kwargs): """ Creates a StreamPublisher which publishes to the specified stream and is attached to the specified process. @param process The IonProcess to attach to. @param stream Name of the stream or StreamRoute object """ super(StreamPublisher, self).__init__() if not isinstance(process, BaseService): raise BadRequest("No valid process provided.") if isinstance(stream, basestring): self.stream_route = StreamRoute(routing_key=stream) elif isinstance(stream, StreamRoute): self.stream_route = stream else: raise BadRequest("No valid stream information provided.") self.container = process.container self.xp_name = get_streaming_xp( self.stream_route.exchange_point) # Fully qualified self.xp = self.container.ex_manager.create_xp( self.stream_route.exchange_point or DEFAULT_DATA_XP) self.xp_route = self.xp.create_route(self.stream_route.routing_key) Publisher.__init__(self, to_name=self.xp_route, **kwargs)
def on_init(self): self.ingestion_profile = self.CFG.get_safe(CONFIG_KEY + ".ingestion_profile", "default") log.info("Ingestion starting using profile '%s'", self.ingestion_profile) self.exchange_name = "ingestion_process" self.ingestion_config = self.CFG.get_safe(CONFIG_KEY + ".profile_" + self.ingestion_profile) or {} if not self.ingestion_config: raise BadRequest("No config found for profile '%s'" % self.ingestion_profile) plugin_cls = get_safe(self.ingestion_config, "plugin") self.plugin = named_any(plugin_cls)(self) log.info("Started ingestion plugin '%s'", plugin_cls) self.persistence_formats = {} self.persistence_objects = {} self.default_persistence_format = get_safe(self.ingestion_config, "persist.persistence_format") self._require_persistence_layer(self.default_persistence_format) self.stream_sub = StreamSubscriber(process=self, exchange_name=self.exchange_name, callback=self.process_package) streams = get_safe(self.ingestion_config, "stream_subscriptions") or [] for stream in streams: if isinstance(stream, list): stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1]) log.info("Ingestion subscribed to stream '%s'", stream) self.stream_sub.add_stream_subscription(stream) self.plugin.on_init() self.stream_sub.start()
def preprocess(self, msg, headers): ''' Performs de-encapsulation of incoming packets and calls the callback. @param msg The incoming packet. @param headers The headers of the incoming message. ''' route = StreamRoute(headers['exchange_point'], headers['routing_key']) self.callback(msg, route, headers['stream'])
def preprocess(self, msg, headers): """ Unwrap the incoming message and calls the callback. @param msg The incoming packet. @param headers The headers of the incoming message. """ route = StreamRoute(headers['exchange_point'], headers['routing_key']) self.callback(msg, route, headers['stream'])
def test_stream_transforms(self): self.verified = Event() input_route = StreamRoute('test_exchange', 'input') output_route = StreamRoute('test_exchange', 'output') def verify(m, route, stream_id): self.assertEquals(route, output_route) self.assertEquals(m, 'test') self.verified.set() # Create I/O Processes #-------------------------------------------------------------------------------- pub_proc = TransformBase() pub_proc.container = self.container publisher = StreamPublisher(process=pub_proc, stream_route=input_route) transform = self.container.spawn_process( 'transform', 'ion.core.process.test.test_transform', 'EmptyDataProcess', { 'process': { 'queue_name': 'transform_input', 'exchange_point': output_route.exchange_point, 'routing_key': output_route.routing_key } }, 'transformpid') transform = self.container.proc_manager.procs[transform] sub_proc = TransformBase() sub_proc.container = self.container subscriber = StreamSubscriber(process=sub_proc, exchange_name='subscriber', callback=verify) # Bind the transports #-------------------------------------------------------------------------------- transform.subscriber.xn.bind(input_route.routing_key, publisher.xp) subscriber.xn.bind(output_route.routing_key, transform.publisher.xp) subscriber.start() self.addCleanup(subscriber.stop) publisher.publish('test') self.assertTrue(self.verified.wait(4))
def add_stream_subscription(self, stream): if isinstance(stream, basestring): stream_route = StreamRoute(routing_key=stream) elif isinstance(stream, StreamRoute): stream_route = stream else: raise BadRequest("No valid stream information provided.") xp = self.container.ex_manager.create_xp(stream_route.exchange_point or DEFAULT_DATA_XP) self.xn.bind(stream_route.routing_key, xp) self.streams.append(stream_route)
def __init__(self, process=None, stream_id='', stream_route=None, exchange_point='', routing_key=''): """ Creates a StreamPublisher which publishes to the specified stream by default and is attached to the specified process. @param process The process which the subscriber is to be attached. @param stream_id Stream identifier for the publishing stream. @param stream_route A StreamRoute corresponding to the stream_id @param exchange_point The name of the exchange point, to be used in lieu of stream_route or stream_id @param routing_key The routing key to be used in lieu of stream_route or stream_id """ super(StreamPublisher, self).__init__() if not isinstance(process, BaseService): raise BadRequest('No valid process provided.') #-------------------------------------------------------------------------------- # The important part of publishing is the stream_route and there are three ways # to the stream route # - The Route is obtained from Pubsub Management with a stream id. # - The Route is obtained by combining exchange_point and the routing_key # but all other information is lost (credentials, etc.) # - The Route is obtained by being provided directly to __init__ #-------------------------------------------------------------------------------- self.stream_id = stream_id if stream_id: # Regardless of what's passed in for stream_route look it up, prevents mismatching pass elif not stream_route: self.stream_route = None if exchange_point and routing_key: self.stream_route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key) else: # Create stream self.stream_id = stream_id self.stream_route = stream_route else: self.stream_route = stream_route if not isinstance(self.stream_route, StreamRoute): raise BadRequest('No valid stream route provided to publisher.') self.container = process.container self.xp = self.container.ex_manager.create_xp( self.stream_route.exchange_point) self.xp_route = self.xp.create_route(self.stream_route.routing_key)
def remove_stream_subscription(self, stream): if isinstance(stream, basestring): stream_route = StreamRoute(routing_key=stream) elif isinstance(stream, StreamRoute): stream_route = stream else: raise BadRequest("No valid stream information provided.") existing_st = None for st in self.streams: if st.routing_key == stream_route.routing_key and st.exchange_point == stream_route.exchange_point: self.streams.remove(st) existing_st = st break if existing_st: xp = get_streaming_xp(stream_route.exchange_point) self.xn.unbind(existing_st.routing_key, xp) else: raise BadRequest("Stream was not a subscription")
def register_producer(self, exchange_name='', stream_id=''): ''' Register a producer with a stream. @param exchange_name The producer exchange name to register. @param stream_id The id of the stream. @retval credentials Credentials for a publisher to use. @throws NotFound when stream doesn't exist. ''' log.debug("Registering producer with stream") stream_obj = self.clients.resource_registry.read(stream_id) if stream_obj is None: raise NotFound("Stream %s does not exist" % stream_id) stream_obj.producers.append(exchange_name) self.update_stream(stream_obj) stream_route_obj = StreamRoute(routing_key=stream_id + '.data') return stream_route_obj
def _construct_publishers(self, stream_info): for (stream_name, stream_config) in stream_info.iteritems(): try: exchange_point = stream_config['exchange_point'] routing_key = stream_config['routing_key'] route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key) stream_id = stream_config['stream_id'] publisher = StreamPublisher(process=self._agent, stream_id=stream_id, stream_route=route) self._publishers[stream_name] = publisher self._stream_greenlets[stream_name] = None self._stream_buffers[stream_name] = [] except Exception as e: errmsg = 'Instrument agent %s' % self._agent._proc_name errmsg += 'error constructing publisher for stream %s. ' % stream_name errmsg += str(e) log.error(errmsg)
def test_ctd_L0_all(self): ''' Test that packets are processed by the ctd_L0_all transform ''' #--------------------------------------------------------------------------------------------- # Launch a ctd transform #--------------------------------------------------------------------------------------------- # Create the process definition process_definition = ProcessDefinition( name='ctd_L0_all', description='For testing ctd_L0_all') process_definition.executable[ 'module'] = 'ion.processes.data.transforms.ctd.ctd_L0_all' process_definition.executable['class'] = 'ctd_L0_all' ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition( process_definition=process_definition) # Build the config config = DotDict() config.process.queue_name = self.exchange_name config.process.exchange_point = self.exchange_point pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub.create_stream_definition( 'ctd_all_stream_def', parameter_dictionary_id=pdict_id) cond_stream_id, _ = self.pubsub.create_stream( 'test_cond', exchange_point='science_data', stream_definition_id=stream_def_id) pres_stream_id, _ = self.pubsub.create_stream( 'test_pres', exchange_point='science_data', stream_definition_id=stream_def_id) temp_stream_id, _ = self.pubsub.create_stream( 'test_temp', exchange_point='science_data', stream_definition_id=stream_def_id) config.process.publish_streams.conductivity = cond_stream_id config.process.publish_streams.pressure = pres_stream_id config.process.publish_streams.temperature = temp_stream_id # Schedule the process pid = self.process_dispatcher.schedule_process( process_definition_id=ctd_transform_proc_def_id, configuration=config) #--------------------------------------------------------------------------------------------- # Create subscribers that will receive the conductivity, temperature and pressure granules from # the ctd transform #--------------------------------------------------------------------------------------------- ar_cond = gevent.event.AsyncResult() def subscriber1(m, r, s): ar_cond.set(m) sub_cond = StandaloneStreamSubscriber('sub_cond', subscriber1) self.addCleanup(sub_cond.stop) ar_temp = gevent.event.AsyncResult() def subscriber2(m, r, s): ar_temp.set(m) sub_temp = StandaloneStreamSubscriber('sub_temp', subscriber2) self.addCleanup(sub_temp.stop) ar_pres = gevent.event.AsyncResult() def subscriber3(m, r, s): ar_pres.set(m) sub_pres = StandaloneStreamSubscriber('sub_pres', subscriber3) self.addCleanup(sub_pres.stop) sub_cond_id = self.pubsub.create_subscription( 'subscription_cond', stream_ids=[cond_stream_id], exchange_name='sub_cond') sub_temp_id = self.pubsub.create_subscription( 'subscription_temp', stream_ids=[temp_stream_id], exchange_name='sub_temp') sub_pres_id = self.pubsub.create_subscription( 'subscription_pres', stream_ids=[pres_stream_id], exchange_name='sub_pres') self.pubsub.activate_subscription(sub_cond_id) self.pubsub.activate_subscription(sub_temp_id) self.pubsub.activate_subscription(sub_pres_id) self.queue_cleanup.append(sub_cond.xn.queue) self.queue_cleanup.append(sub_temp.xn.queue) self.queue_cleanup.append(sub_pres.xn.queue) sub_cond.start() sub_temp.start() sub_pres.start() #------------------------------------------------------------------------------------------------------ # Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform #------------------------------------------------------------------------------------------------------ # Do all the routing stuff for the publishing routing_key = 'stream_id.stream' stream_route = StreamRoute(self.exchange_point, routing_key) xn = self.container.ex_manager.create_xn_queue(self.exchange_name) xp = self.container.ex_manager.create_xp(self.exchange_point) xn.bind('stream_id.stream', xp) pub = StandaloneStreamPublisher('stream_id', stream_route) # Build a packet that can be published publish_granule = self._get_new_ctd_packet( stream_definition_id=stream_def_id, length=5) # Publish the packet pub.publish(publish_granule) #------------------------------------------------------------------------------------------------------ # Make assertions about whether the ctd transform executed its algorithm and published the correct # granules #------------------------------------------------------------------------------------------------------ # Get the granule that is published by the ctd transform post processing result_cond = ar_cond.get(timeout=10) result_temp = ar_temp.get(timeout=10) result_pres = ar_pres.get(timeout=10) out_dict = {} out_dict['c'] = RecordDictionaryTool.load_from_granule( result_cond)['conductivity'] out_dict['t'] = RecordDictionaryTool.load_from_granule( result_temp)['temp'] out_dict['p'] = RecordDictionaryTool.load_from_granule( result_pres)['pressure'] # Check that the transform algorithm was successfully executed self.check_granule_splitting(publish_granule, out_dict)
def test_event_triggered_transform_A(self): ''' Test that packets are processed by the event triggered transform ''' #--------------------------------------------------------------------------------------------- # Launch a ctd transform #--------------------------------------------------------------------------------------------- # Create the process definition process_definition = ProcessDefinition( name='EventTriggeredTransform_A', description='For testing EventTriggeredTransform_A') process_definition.executable[ 'module'] = 'ion.processes.data.transforms.event_triggered_transform' process_definition.executable['class'] = 'EventTriggeredTransform_A' event_transform_proc_def_id = self.process_dispatcher.create_process_definition( process_definition=process_definition) # Build the config config = DotDict() config.process.queue_name = self.exchange_name config.process.exchange_point = self.exchange_point pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub.create_stream_definition( 'cond_stream_def', parameter_dictionary_id=pdict_id) cond_stream_id, _ = self.pubsub.create_stream( 'test_conductivity', exchange_point='science_data', stream_definition_id=stream_def_id) config.process.publish_streams.conductivity = cond_stream_id config.process.event_type = 'ResourceLifecycleEvent' # Schedule the process self.process_dispatcher.schedule_process( process_definition_id=event_transform_proc_def_id, configuration=config) #--------------------------------------------------------------------------------------------- # Publish an event to wake up the event triggered transform #--------------------------------------------------------------------------------------------- event_publisher = EventPublisher("ResourceLifecycleEvent") event_publisher.publish_event(origin='fake_origin') #--------------------------------------------------------------------------------------------- # Create subscribers that will receive the conductivity, temperature and pressure granules from # the ctd transform #--------------------------------------------------------------------------------------------- ar_cond = gevent.event.AsyncResult() def subscriber1(m, r, s): ar_cond.set(m) sub_event_transform = StandaloneStreamSubscriber( 'sub_event_transform', subscriber1) self.addCleanup(sub_event_transform.stop) sub_event_transform_id = self.pubsub.create_subscription( 'subscription_cond', stream_ids=[cond_stream_id], exchange_name='sub_event_transform') self.pubsub.activate_subscription(sub_event_transform_id) self.queue_cleanup.append(sub_event_transform.xn.queue) sub_event_transform.start() #------------------------------------------------------------------------------------------------------ # Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform #------------------------------------------------------------------------------------------------------ # Do all the routing stuff for the publishing routing_key = 'stream_id.stream' stream_route = StreamRoute(self.exchange_point, routing_key) xn = self.container.ex_manager.create_xn_queue(self.exchange_name) xp = self.container.ex_manager.create_xp(self.exchange_point) xn.bind('stream_id.stream', xp) pub = StandaloneStreamPublisher('stream_id', stream_route) # Build a packet that can be published self.px_ctd = SimpleCtdPublisher() publish_granule = self._get_new_ctd_packet( stream_definition_id=stream_def_id, length=5) # Publish the packet pub.publish(publish_granule) #------------------------------------------------------------------------------------------------------ # Make assertions about whether the ctd transform executed its algorithm and published the correct # granules #------------------------------------------------------------------------------------------------------ # Get the granule that is published by the ctd transform post processing result_cond = ar_cond.get(timeout=10) self.assertTrue(isinstance(result_cond, Granule)) rdt = RecordDictionaryTool.load_from_granule(result_cond) self.assertTrue(rdt.__contains__('conductivity')) self.check_cond_algorithm_execution(publish_granule, result_cond)
def test_stream_processing(self): #-------------------------------------------------------------------------------- #Test that streams are processed by the transforms according to a provided algorithm #-------------------------------------------------------------------------------- #todo: In this simple implementation, we are checking if the stream has the word, PUBLISH, #todo(contd) and if the word VALUE=<number> exists and that number is less than something #todo later on we are going to use complex algorithms to make this prototype powerful #------------------------------------------------------------------------------------- # Start a subscriber to listen for an alert event from the Stream Alert Transform #------------------------------------------------------------------------------------- queue = gevent.queue.Queue() def event_received(message, headers): queue.put(message) event_subscriber = EventSubscriber( origin="StreamAlertTransform", event_type="DeviceEvent", callback=event_received) event_subscriber.start() self.addCleanup(event_subscriber.stop) #------------------------------------------------------------------------------------- # The configuration for the Stream Alert Transform... set up the event types to listen to #------------------------------------------------------------------------------------- config = { 'process':{ 'queue_name': 'a_queue', 'value': 10, 'event_type':'DeviceEvent' } } #------------------------------------------------------------------------------------- # Create the process #------------------------------------------------------------------------------------- pid = TransformPrototypeIntTest.create_process( name= 'transform_data_process', module='ion.processes.data.transforms.event_alert_transform', class_name='StreamAlertTransform', configuration= config) self.addCleanup(self.process_dispatcher.cancel_process, pid) self.assertIsNotNone(pid) #------------------------------------------------------------------------------------- # Publish streams and make assertions about alerts #------------------------------------------------------------------------------------- exchange_name = 'a_queue' exchange_point = 'test_exchange' routing_key = 'stream_id.stream' stream_route = StreamRoute(exchange_point, routing_key) xn = self.container.ex_manager.create_xn_queue(exchange_name) xp = self.container.ex_manager.create_xp(exchange_point) xn.bind('stream_id.stream', xp) pub = StandaloneStreamPublisher('stream_id', stream_route) message = "A dummy example message containing the word PUBLISH, and with VALUE = 5 . This message" +\ " will trigger an alert event from the StreamAlertTransform because the value provided is "\ "less than 10 that was passed in through the config." pub.publish(message) event = queue.get(timeout=10) self.assertEquals(event.type_, "DeviceEvent") self.assertEquals(event.origin, "StreamAlertTransform")
def test_ctd_L2_salinity(self): ''' Test that packets are processed by the ctd_L1_salinity transform ''' #--------------------------------------------------------------------------------------------- # Launch a ctd transform #--------------------------------------------------------------------------------------------- # Create the process definition process_definition = ProcessDefinition( name='SalinityTransform', description='For testing SalinityTransform') process_definition.executable[ 'module'] = 'ion.processes.data.transforms.ctd.ctd_L2_salinity' process_definition.executable['class'] = 'SalinityTransform' ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition( process_definition=process_definition) # Build the config config = DotDict() config.process.queue_name = self.exchange_name config.process.exchange_point = self.exchange_point pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub.create_stream_definition( 'sal_stream_def', parameter_dictionary_id=pdict_id) sal_stream_id, _ = self.pubsub.create_stream( 'test_salinity', stream_definition_id=stream_def_id, exchange_point='science_data') config.process.publish_streams.salinity = sal_stream_id # Schedule the process self.process_dispatcher.schedule_process( process_definition_id=ctd_transform_proc_def_id, configuration=config) #--------------------------------------------------------------------------------------------- # Create a subscriber that will receive the salinity granule from the ctd transform #--------------------------------------------------------------------------------------------- ar_sal = gevent.event.AsyncResult() def subscriber3(m, r, s): ar_sal.set(m) sub_sal = StandaloneStreamSubscriber('sub_sal', subscriber3) self.addCleanup(sub_sal.stop) sub_sal_id = self.pubsub.create_subscription( 'subscription_sal', stream_ids=[sal_stream_id], exchange_name='sub_sal') self.pubsub.activate_subscription(sub_sal_id) self.queue_cleanup.append(sub_sal.xn.queue) sub_sal.start() #------------------------------------------------------------------------------------------------------ # Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform #------------------------------------------------------------------------------------------------------ # Do all the routing stuff for the publishing routing_key = 'stream_id.stream' stream_route = StreamRoute(self.exchange_point, routing_key) xn = self.container.ex_manager.create_xn_queue(self.exchange_name) xp = self.container.ex_manager.create_xp(self.exchange_point) xn.bind('stream_id.stream', xp) pub = StandaloneStreamPublisher('stream_id', stream_route) # Build a packet that can be published publish_granule = self._get_new_ctd_packet( stream_definition_id=stream_def_id, length=5) # Publish the packet pub.publish(publish_granule) #------------------------------------------------------------------------------------------------------ # Make assertions about whether the ctd transform executed its algorithm and published the correct # granules #------------------------------------------------------------------------------------------------------ # Get the granule that is published by the ctd transform post processing result = ar_sal.get(timeout=10) self.assertTrue(isinstance(result, Granule)) rdt = RecordDictionaryTool.load_from_granule(result) self.assertTrue(rdt.__contains__('salinity')) self.check_salinity_algorithm_execution(publish_granule, result)