def test_stream_pub_sub(self):
self.verified = Event()
self.route = StreamRoute(exchange_point='xp_test', routing_key='route')
def verify(message, route, stream):
self.assertEquals(message,'test')
self.assertEquals(route, self.route)
self.assertEquals(stream, '')
self.verified.set()
sub_proc = SimpleProcess()
sub_proc.container = self.container
sub1 = StreamSubscriber(process=sub_proc, exchange_name='sub1', callback=verify)
sub1.start()
self.queue_cleanup.append('sub1')
pub_proc = SimpleProcess()
pub_proc.container = self.container
pub1 = StreamPublisher(process=pub_proc,stream_route=self.route)
sub1.xn.bind(self.route.routing_key,pub1.xp)
pub1.publish('test')
self.assertTrue(self.verified.wait(2))
def test_stream_pub_sub(self):
self.verified = Event()
self.route = StreamRoute(routing_key='stream_name')
def verify(message, route, stream):
self.assertEquals(message, 'test')
self.assertEquals(route.routing_key, self.route.routing_key)
self.assertTrue(route.exchange_point.startswith(get_sys_name()))
self.assertEquals(stream, 'stream_name')
self.verified.set()
sub_proc = SimpleProcess()
sub_proc.container = self.container
sub1 = StreamSubscriber(process=sub_proc, exchange_name='stream_name', callback=verify)
sub1.add_stream_subscription("stream_name")
sub1.start()
self.queue_cleanup.append('data.stream_name')
pub_proc = SimpleProcess()
pub_proc.container = self.container
pub1 = StreamPublisher(process=pub_proc, stream=self.route)
sub1.xn.bind(self.route.routing_key, pub1.xp)
pub1.publish('test')
self.assertTrue(self.verified.wait(2))
def test_stream_pub_sub(self):
self.verified = Event()
self.route = StreamRoute(exchange_point='xp_test', routing_key='route')
def verify(message, route, stream):
self.assertEquals(message, 'test')
self.assertEquals(route, self.route)
self.assertEquals(stream, '')
self.verified.set()
sub_proc = SimpleProcess()
sub_proc.container = self.container
sub1 = StreamSubscriber(process=sub_proc,
exchange_name='sub1',
callback=verify)
sub1.start()
self.queue_cleanup.append('sub1')
pub_proc = SimpleProcess()
pub_proc.container = self.container
pub1 = StreamPublisher(process=pub_proc, stream_route=self.route)
sub1.xn.bind(self.route.routing_key, pub1.xp)
pub1.publish('test')
self.assertTrue(self.verified.wait(2))
def test_stream_pub_sub(self):
self.verified = Event()
self.route = StreamRoute(routing_key='stream_name')
def verify(message, route, stream):
self.assertEquals(message, 'test')
self.assertEquals(route.routing_key, self.route.routing_key)
self.assertTrue(route.exchange_point.startswith(get_sys_name()))
self.assertEquals(stream, 'stream_name')
self.verified.set()
sub_proc = SimpleProcess()
sub_proc.container = self.container
sub1 = StreamSubscriber(process=sub_proc,
exchange_name='stream_name',
callback=verify)
sub1.add_stream_subscription("stream_name")
sub1.start()
self.queue_cleanup.append('data.stream_name')
pub_proc = SimpleProcess()
pub_proc.container = self.container
pub1 = StreamPublisher(process=pub_proc, stream=self.route)
sub1.xn.bind(self.route.routing_key, pub1.xp)
pub1.publish('test')
self.assertTrue(self.verified.wait(2))
def test_stream_transforms(self):
self.verified = Event()
input_route = StreamRoute('test_exchange', 'input')
output_route = StreamRoute('test_exchange', 'output')
def verify(m, route, stream_id):
self.assertEquals(route, output_route)
self.assertEquals(m, 'test')
self.verified.set()
# Create I/O Processes
#--------------------------------------------------------------------------------
pub_proc = TransformBase()
pub_proc.container = self.container
publisher = StreamPublisher(process=pub_proc, stream_route=input_route)
transform = self.container.spawn_process(
'transform', 'ion.core.process.test.test_transform',
'EmptyDataProcess', {
'process': {
'queue_name': 'transform_input',
'exchange_point': output_route.exchange_point,
'routing_key': output_route.routing_key
}
}, 'transformpid')
transform = self.container.proc_manager.procs[transform]
sub_proc = TransformBase()
sub_proc.container = self.container
subscriber = StreamSubscriber(process=sub_proc,
exchange_name='subscriber',
callback=verify)
# Bind the transports
#--------------------------------------------------------------------------------
transform.subscriber.xn.bind(input_route.routing_key, publisher.xp)
subscriber.xn.bind(output_route.routing_key, transform.publisher.xp)
subscriber.start()
self.addCleanup(subscriber.stop)
publisher.publish('test')
self.assertTrue(self.verified.wait(4))
def _on_done(self):
'''
Callback for the thread when the query is complete.
Iterate through the entries and publish each post and comment(s) on an independent stream
'''
num=0
for entry in self.entries:
"""
We are skipping create stream & register producer here. Create stream should not be called by a steam
producing process such as an instrument driver or data agent. It should be called at a higher level.
For the reason we will pretend it has already been called and use an unregistered stream.
"""
p = StreamPublisher(name=(self.XP,'%s.%s' %(num,"data")),process=self,node=self.container.node)
p.publish(msg=entry['post'])
log.debug('Published post id %s' % entry['post'].post_id)
for comment in entry['comments']:
p.publish(msg=comment)
num+=1
log.info('Completed Publishing Blog Results for Blog %s' % self.feed.blog)
def test_stream_transforms(self):
self.verified = Event()
input_route = StreamRoute('test_exchange','input')
output_route = StreamRoute('test_exchange','output')
def verify(m, route, stream_id):
self.assertEquals(route,output_route)
self.assertEquals(m,'test')
self.verified.set()
# Create I/O Processes
#--------------------------------------------------------------------------------
pub_proc = TransformBase()
pub_proc.container = self.container
publisher = StreamPublisher(process=pub_proc, stream_route=input_route)
transform = self.container.spawn_process('transform','ion.core.process.test.test_transform','EmptyDataProcess',{'process':{'queue_name':'transform_input', 'exchange_point':output_route.exchange_point, 'routing_key':output_route.routing_key}}, 'transformpid')
transform = self.container.proc_manager.procs[transform]
sub_proc = TransformBase()
sub_proc.container = self.container
subscriber = StreamSubscriber(process=sub_proc, exchange_name='subscriber', callback=verify)
# Bind the transports
#--------------------------------------------------------------------------------
transform.subscriber.xn.bind(input_route.routing_key, publisher.xp)
subscriber.xn.bind(output_route.routing_key, transform.publisher.xp)
subscriber.start()
self.addCleanup(subscriber.stop)
publisher.publish('test')
self.assertTrue(self.verified.wait(4))
def _on_done(self):
'''
Callback for the thread when the query is complete.
Iterate through the entries and publish each post and comment(s) on an independent stream
'''
num = 0
for entry in self.entries:
"""
We are skipping create stream & register producer here. Create stream should not be called by a steam
producing process such as an instrument driver or data agent. It should be called at a higher level.
For the reason we will pretend it has already been called and use an unregistered stream.
"""
p = StreamPublisher(name=(self.XP, '%s.%s' % (num, "data")),
process=self,
node=self.container.node)
p.publish(msg=entry['post'])
log.debug('Published post id %s' % entry['post'].post_id)
for comment in entry['comments']:
p.publish(msg=comment)
num += 1
log.info('Completed Publishing Blog Results for Blog %s' %
self.feed.blog)