class TransformStreamListener(TransformStreamProcess): ''' Transforms which listen to a queue for incoming Ion Streams. Parameters: process.queue_name Name of the queue to listen on. ''' def __init__(self): super(TransformStreamListener,self).__init__() def on_start(self): ''' Sets up the subscribing endpoint and begins consuming. ''' super(TransformStreamListener,self).on_start() self.queue_name = self.CFG.get_safe('process.queue_name',self.id) self.subscriber = StreamSubscriber(process=self, exchange_name=self.queue_name, callback=self.recv_packet) self.subscriber.start() def recv_packet(self, msg, stream_route, stream_id): ''' To be implemented by the transform developer. This method is called on receipt of an incoming message from a stream. ''' raise NotImplementedError('Method recv_packet not implemented') def on_quit(self): ''' Stops consuming on the queue. ''' self.subscriber.stop() super(TransformStreamListener,self).on_quit()
def test_stream_pub_sub(self): self.verified = Event() self.route = StreamRoute(exchange_point='xp_test', routing_key='route') def verify(message, route, stream): self.assertEquals(message, 'test') self.assertEquals(route, self.route) self.assertEquals(stream, '') self.verified.set() sub_proc = SimpleProcess() sub_proc.container = self.container sub1 = StreamSubscriber(process=sub_proc, exchange_name='sub1', callback=verify) sub1.start() self.queue_cleanup.append('sub1') pub_proc = SimpleProcess() pub_proc.container = self.container pub1 = StreamPublisher(process=pub_proc, stream_route=self.route) sub1.xn.bind(self.route.routing_key, pub1.xp) pub1.publish('test') self.assertTrue(self.verified.wait(2))
def test_stream_pub_sub(self): self.verified = Event() self.route = StreamRoute(exchange_point='xp_test', routing_key='route') def verify(message, route, stream): self.assertEquals(message,'test') self.assertEquals(route, self.route) self.assertEquals(stream, '') self.verified.set() sub_proc = SimpleProcess() sub_proc.container = self.container sub1 = StreamSubscriber(process=sub_proc, exchange_name='sub1', callback=verify) sub1.start() self.queue_cleanup.append('sub1') pub_proc = SimpleProcess() pub_proc.container = self.container pub1 = StreamPublisher(process=pub_proc,stream_route=self.route) sub1.xn.bind(self.route.routing_key,pub1.xp) pub1.publish('test') self.assertTrue(self.verified.wait(2))
def test_stream_pub_sub(self): self.verified = Event() self.route = StreamRoute(routing_key='stream_name') def verify(message, route, stream): self.assertEquals(message, 'test') self.assertEquals(route.routing_key, self.route.routing_key) self.assertTrue(route.exchange_point.startswith(get_sys_name())) self.assertEquals(stream, 'stream_name') self.verified.set() sub_proc = SimpleProcess() sub_proc.container = self.container sub1 = StreamSubscriber(process=sub_proc, exchange_name='stream_name', callback=verify) sub1.add_stream_subscription("stream_name") sub1.start() self.queue_cleanup.append('data.stream_name') pub_proc = SimpleProcess() pub_proc.container = self.container pub1 = StreamPublisher(process=pub_proc, stream=self.route) sub1.xn.bind(self.route.routing_key, pub1.xp) pub1.publish('test') self.assertTrue(self.verified.wait(2))
def test_stream_pub_sub(self): self.verified = Event() self.route = StreamRoute(routing_key='stream_name') def verify(message, route, stream): self.assertEquals(message, 'test') self.assertEquals(route.routing_key, self.route.routing_key) self.assertTrue(route.exchange_point.startswith(get_sys_name())) self.assertEquals(stream, 'stream_name') self.verified.set() sub_proc = SimpleProcess() sub_proc.container = self.container sub1 = StreamSubscriber(process=sub_proc, exchange_name='stream_name', callback=verify) sub1.add_stream_subscription("stream_name") sub1.start() self.queue_cleanup.append('data.stream_name') pub_proc = SimpleProcess() pub_proc.container = self.container pub1 = StreamPublisher(process=pub_proc, stream=self.route) sub1.xn.bind(self.route.routing_key, pub1.xp) pub1.publish('test') self.assertTrue(self.verified.wait(2))
def on_start(self): rr_cli = ResourceRegistryServiceProcessClient(process=self, node=self.container.node) pubsub_cli = PubsubManagementServiceProcessClient(process=self, node=self.container.node) # Get the stream(s) data_product_id = self.CFG.get_safe('dispatcher.data_product_id','') stream_ids,_ = rr_cli.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) log.info('Got Stream Ids: "%s"', stream_ids) assert stream_ids, 'No streams found for this data product!' exchange_name = 'dispatcher_%s' % str(os.getpid()) subscription_id = pubsub_cli.create_subscription( name='SampleSubscription', exchange_name=exchange_name, stream_ids=stream_ids, description='Sample Subscription Description' ) stream_defs = {} def message_received(message, stream_route, stream_id): granule = message stream_id = granule.stream_resource_id data_stream_id = granule.data_stream_id data_stream = granule.identifiables[data_stream_id] tstamp = get_datetime(data_stream.timestamp.value) records = granule.identifiables['record_count'].value log.info('Received a message from stream %s with time stamp %s and %d records' % (stream_id, tstamp, records)) if stream_id not in stream_defs: stream_defs[stream_id] = pubsub_cli.find_stream_definition(stream_id, id_only=False).container stream_def = stream_defs.get(stream_id) sp = PointSupplementStreamParser(stream_definition=stream_def, stream_granule=granule) last_data = {} for field in sp.list_field_names(): last_data[field] = sp.get_values(field)[-1] log.info('Last values in the message: %s' % str(last_data)) subscriber = StreamSubscriber(process=self, exchange_name=exchange_name, callback=message_received) subscriber.start() pubsub_cli.activate_subscription(subscription_id)
class TransformStreamListener(TransformStreamProcess): ''' Transforms which listen to a queue for incoming Ion Streams. Parameters: process.queue_name Name of the queue to listen on. ''' def __init__(self): super(TransformStreamListener,self).__init__() def on_start(self): ''' Sets up the subscribing endpoint and begins consuming. ''' super(TransformStreamListener,self).on_start() self.queue_name = self.CFG.get_safe('process.queue_name',self.id) self.subscriber = StreamSubscriber(process=self, exchange_name=self.queue_name, callback=self.receive_callback) self.subscriber.start() def receive_callback(self, *a, **b): """ wrapper to capture errors """ try: self.recv_packet(*a,**b) except: # try to get the individual transform's logger try: log = logging.getLogger(self.__class__.__module__) except: pass # but settle for ion.core.process.transform's logger if you have to log.warning('transform %s failed to handle message', self.id, exc_info=True) raise def recv_packet(self, msg, stream_route, stream_id): ''' To be implemented by the transform developer. This method is called on receipt of an incoming message from a stream. ''' raise NotImplementedError('Method recv_packet not implemented') def on_quit(self): ''' Stops consuming on the queue. ''' self.subscriber.stop() super(TransformStreamListener,self).on_quit()
def test_stream_transforms(self): self.verified = Event() input_route = StreamRoute('test_exchange', 'input') output_route = StreamRoute('test_exchange', 'output') def verify(m, route, stream_id): self.assertEquals(route, output_route) self.assertEquals(m, 'test') self.verified.set() # Create I/O Processes #-------------------------------------------------------------------------------- pub_proc = TransformBase() pub_proc.container = self.container publisher = StreamPublisher(process=pub_proc, stream_route=input_route) transform = self.container.spawn_process( 'transform', 'ion.core.process.test.test_transform', 'EmptyDataProcess', { 'process': { 'queue_name': 'transform_input', 'exchange_point': output_route.exchange_point, 'routing_key': output_route.routing_key } }, 'transformpid') transform = self.container.proc_manager.procs[transform] sub_proc = TransformBase() sub_proc.container = self.container subscriber = StreamSubscriber(process=sub_proc, exchange_name='subscriber', callback=verify) # Bind the transports #-------------------------------------------------------------------------------- transform.subscriber.xn.bind(input_route.routing_key, publisher.xp) subscriber.xn.bind(output_route.routing_key, transform.publisher.xp) subscriber.start() self.addCleanup(subscriber.stop) publisher.publish('test') self.assertTrue(self.verified.wait(4))
def test_stream_transforms(self): self.verified = Event() input_route = StreamRoute('test_exchange','input') output_route = StreamRoute('test_exchange','output') def verify(m, route, stream_id): self.assertEquals(route,output_route) self.assertEquals(m,'test') self.verified.set() # Create I/O Processes #-------------------------------------------------------------------------------- pub_proc = TransformBase() pub_proc.container = self.container publisher = StreamPublisher(process=pub_proc, stream_route=input_route) transform = self.container.spawn_process('transform','ion.core.process.test.test_transform','EmptyDataProcess',{'process':{'queue_name':'transform_input', 'exchange_point':output_route.exchange_point, 'routing_key':output_route.routing_key}}, 'transformpid') transform = self.container.proc_manager.procs[transform] sub_proc = TransformBase() sub_proc.container = self.container subscriber = StreamSubscriber(process=sub_proc, exchange_name='subscriber', callback=verify) # Bind the transports #-------------------------------------------------------------------------------- transform.subscriber.xn.bind(input_route.routing_key, publisher.xp) subscriber.xn.bind(output_route.routing_key, transform.publisher.xp) subscriber.start() self.addCleanup(subscriber.stop) publisher.publish('test') self.assertTrue(self.verified.wait(4))
class TransformStreamListener(TransformStreamProcess): ''' Transforms which listen to a queue for incoming Ion Streams. Parameters: process.queue_name Name of the queue to listen on. ''' def __init__(self): super(TransformStreamListener, self).__init__() def on_start(self): ''' Sets up the subscribing endpoint and begins consuming. ''' super(TransformStreamListener, self).on_start() self.queue_name = self.CFG.get_safe('process.queue_name', self.id) self.subscriber = StreamSubscriber(process=self, exchange_name=self.queue_name, callback=self.recv_packet) self.subscriber.start() def recv_packet(self, msg, stream_route, stream_id): ''' To be implemented by the transform developer. This method is called on receipt of an incoming message from a stream. ''' raise NotImplementedError('Method recv_packet not implemented') def on_quit(self): ''' Stops consuming on the queue. ''' self.subscriber.stop() super(TransformStreamListener, self).on_quit()