def test_stream_pub_sub(self):
self.verified = Event()
self.route = StreamRoute(exchange_point='xp_test', routing_key='route')
def verify(message, route, stream):
self.assertEquals(message, 'test')
self.assertEquals(route, self.route)
self.assertEquals(stream, '')
self.verified.set()
sub_proc = SimpleProcess()
sub_proc.container = self.container
sub1 = StreamSubscriber(process=sub_proc,
exchange_name='sub1',
callback=verify)
sub1.start()
self.queue_cleanup.append('sub1')
pub_proc = SimpleProcess()
pub_proc.container = self.container
pub1 = StreamPublisher(process=pub_proc, stream_route=self.route)
sub1.xn.bind(self.route.routing_key, pub1.xp)
pub1.publish('test')
self.assertTrue(self.verified.wait(2))
def on_start(self): #pragma no cover
#--------------------------------------------------------------------------------
# Explicit on_start
#--------------------------------------------------------------------------------
# Skip TransformStreamListener and go to StreamProcess to avoid the subscriber being created
# We want explicit management of the thread and subscriber object for ingestion
TransformStreamProcess.on_start(self)
self.queue_name = self.CFG.get_safe('process.queue_name',self.id)
self.subscriber = StreamSubscriber(process=self, exchange_name=self.queue_name, callback=self.receive_callback)
self.thread_lock = RLock()
#--------------------------------------------------------------------------------
# Normal on_start after this point
#--------------------------------------------------------------------------------
BaseIngestionWorker.on_start(self)
self._rpc_server = self.container.proc_manager._create_listening_endpoint(from_name=self.id, process=self)
self.add_endpoint(self._rpc_server)
self.event_publisher = EventPublisher(OT.DatasetModified)
self.stored_value_manager = StoredValueManager(self.container)
self.lookup_docs = self.CFG.get_safe('process.lookup_docs',[])
self.input_product = self.CFG.get_safe('process.input_product','')
self.new_lookups = Queue()
self.lookup_monitor = EventSubscriber(event_type=OT.ExternalReferencesUpdatedEvent, callback=self._add_lookups, auto_delete=True)
self.add_endpoint(self.lookup_monitor)
self.connection_id = ''
self.connection_index = None
self.start_listener()
def on_start(self): #pragma no cover
#super(TransformWorker,self).on_start()
#--------------------------------------------------------------------------------
# Explicit on_start
#--------------------------------------------------------------------------------
# Skip TransformStreamListener and go to StreamProcess to avoid the subscriber being created
# We want explicit management of the thread and subscriber object for ingestion
#todo: check how to manage multi queue subscription (transform scenario 3)
TransformStreamProcess.on_start(self)
#todo: can the subscription be changed or updated when new dataprocesses are added ?
self.queue_name = self.CFG.get_safe('process.queue_name', self.id)
self.subscriber = StreamSubscriber(process=self,
exchange_name=self.queue_name,
callback=self.receive_callback)
self.thread_lock = RLock()
self._rpc_server = self.container.proc_manager._create_listening_endpoint(
from_name=self.id, process=self)
self.add_endpoint(self._rpc_server)
self.start_listener()
#todo: determine and publish appropriate set of status events
self.event_publisher = EventPublisher(OT.DataProcessStatusEvent)
def test_stream_pub_sub(self):
self.verified = Event()
self.route = StreamRoute(routing_key='stream_name')
def verify(message, route, stream):
self.assertEquals(message, 'test')
self.assertEquals(route.routing_key, self.route.routing_key)
self.assertTrue(route.exchange_point.startswith(get_sys_name()))
self.assertEquals(stream, 'stream_name')
self.verified.set()
sub_proc = SimpleProcess()
sub_proc.container = self.container
sub1 = StreamSubscriber(process=sub_proc,
exchange_name='stream_name',
callback=verify)
sub1.add_stream_subscription("stream_name")
sub1.start()
self.queue_cleanup.append('data.stream_name')
pub_proc = SimpleProcess()
pub_proc.container = self.container
pub1 = StreamPublisher(process=pub_proc, stream=self.route)
sub1.xn.bind(self.route.routing_key, pub1.xp)
pub1.publish('test')
self.assertTrue(self.verified.wait(2))
def on_start(self):
rr_cli = ResourceRegistryServiceProcessClient(process=self, node=self.container.node)
pubsub_cli = PubsubManagementServiceProcessClient(process=self, node=self.container.node)
# Get the stream(s)
data_product_id = self.CFG.get_safe('dispatcher.data_product_id','')
stream_ids,_ = rr_cli.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
log.info('Got Stream Ids: "%s"', stream_ids)
assert stream_ids, 'No streams found for this data product!'
exchange_name = 'dispatcher_%s' % str(os.getpid())
subscription_id = pubsub_cli.create_subscription(
name='SampleSubscription',
exchange_name=exchange_name,
stream_ids=stream_ids,
description='Sample Subscription Description'
)
stream_defs = {}
def message_received(message, stream_route, stream_id):
granule = message
stream_id = granule.stream_resource_id
data_stream_id = granule.data_stream_id
data_stream = granule.identifiables[data_stream_id]
tstamp = get_datetime(data_stream.timestamp.value)
records = granule.identifiables['record_count'].value
log.info('Received a message from stream %s with time stamp %s and %d records' % (stream_id, tstamp, records))
if stream_id not in stream_defs:
stream_defs[stream_id] = pubsub_cli.find_stream_definition(stream_id, id_only=False).container
stream_def = stream_defs.get(stream_id)
sp = PointSupplementStreamParser(stream_definition=stream_def, stream_granule=granule)
last_data = {}
for field in sp.list_field_names():
last_data[field] = sp.get_values(field)[-1]
log.info('Last values in the message: %s' % str(last_data))
subscriber = StreamSubscriber(process=self, exchange_name=exchange_name, callback=message_received)
subscriber.start()
pubsub_cli.activate_subscription(subscription_id)
def on_start(self):
'''
Sets up the subscribing endpoint and begins consuming.
'''
super(TransformStreamListener, self).on_start()
self.queue_name = self.CFG.get_safe('process.queue_name', self.id)
self.subscriber = StreamSubscriber(process=self,
exchange_name=self.queue_name,
callback=self.receive_callback)
self.add_endpoint(self.subscriber)
def test_stream_transforms(self):
self.verified = Event()
input_route = StreamRoute('test_exchange', 'input')
output_route = StreamRoute('test_exchange', 'output')
def verify(m, route, stream_id):
self.assertEquals(route, output_route)
self.assertEquals(m, 'test')
self.verified.set()
# Create I/O Processes
#--------------------------------------------------------------------------------
pub_proc = TransformBase()
pub_proc.container = self.container
publisher = StreamPublisher(process=pub_proc, stream_route=input_route)
transform = self.container.spawn_process(
'transform', 'ion.core.process.test.test_transform',
'EmptyDataProcess', {
'process': {
'queue_name': 'transform_input',
'exchange_point': output_route.exchange_point,
'routing_key': output_route.routing_key
}
}, 'transformpid')
transform = self.container.proc_manager.procs[transform]
sub_proc = TransformBase()
sub_proc.container = self.container
subscriber = StreamSubscriber(process=sub_proc,
exchange_name='subscriber',
callback=verify)
# Bind the transports
#--------------------------------------------------------------------------------
transform.subscriber.xn.bind(input_route.routing_key, publisher.xp)
subscriber.xn.bind(output_route.routing_key, transform.publisher.xp)
subscriber.start()
self.addCleanup(subscriber.stop)
publisher.publish('test')
self.assertTrue(self.verified.wait(4))
def on_start(self):
self.queue_name = self.CFG.get_safe('process.queue_name', self.id)
self.subscriber = StreamSubscriber(process=self,
exchange_name=self.queue_name,
callback=self.recv_packet)
self._msg_buffer = []
self._fsm = FSM(self.S_INIT)
self._fsm.add_transition(self.I_READY, self.S_INIT, None, self.S_READY)
self._fsm.add_transition(self.I_FETCH, self.S_READY, None,
self.S_FETCH)
self._fsm.add_transition(self.I_READY, self.S_FETCH, self._reset,
self.S_READY)
self._fsm.add_transition(self.I_EXCEPTION, self.S_FETCH, None,
self.S_EXCEPTION)
self._fsm.add_transition(self.I_READY, self.S_EXCEPTION, self._reset,
self.S_READY)
self.subscriber.initialize()
self.done = gevent.event.Event()
self._fsm.process(self.I_READY)
self.greenlet = gevent.spawn(self.activate)
def _spawn_stream_process(self, process_id, name, module, cls, config):
"""
Spawn a process acting as a data stream process.
Attach to subscription queue with process function.
"""
process_instance = self._create_process_instance(
process_id, name, module, cls, config)
listen_name = get_safe(config, "process.listen_name") or name
log.debug("Stream Process (%s) listen_name: %s", name, listen_name)
process_instance._proc_listen_name = listen_name
process_instance.stream_subscriber = StreamSubscriber(
process=process_instance,
exchange_name=listen_name,
callback=process_instance.call_process)
# Add publishers if any...
publish_streams = get_safe(config, "process.publish_streams")
pub_names = self._set_publisher_endpoints(process_instance,
publish_streams)
rsvc = self._create_listening_endpoint(node=self.container.node,
from_name=process_instance.id,
service=process_instance,
process=process_instance)
# cleanup method to delete process queue (@TODO: leaks a bit here - should use XOs)
def cleanup(*args):
self._cleanup_method(process_instance.id, rsvc)
for name in pub_names:
p = getattr(process_instance, name)
p.close()
proc = self.proc_sup.spawn(
name=process_instance.id,
service=process_instance,
listeners=[rsvc, process_instance.stream_subscriber],
proc_name=process_instance._proc_name,
cleanup_method=cleanup)
self.proc_sup.ensure_ready(
proc, "_spawn_stream_process for %s" % process_instance._proc_name)
# map gproc to process_instance
self._spawned_proc_to_process[proc.proc] = process_instance
# set service's reference to process
process_instance._process = proc
self._process_init(process_instance)
self._process_start(process_instance)
try:
proc.start_listeners()
except IonProcessError:
self._process_quit(process_instance)
self._call_proc_state_changed(process_instance,
ProcessStateEnum.FAILED)
raise
return process_instance
