def do_listen_for_incoming(self):
subscription_id = self.pubsub.create_subscription(
'validator', data_product_ids=[self.data_product._id])
self.addCleanup(self.pubsub.delete_subscription, subscription_id)
self.granule_capture = []
self.granule_count = 0
def on_granule(msg, route, stream_id):
self.granule_count += 1
if self.granule_count < 5:
self.granule_capture.append(msg)
validator = StandaloneStreamSubscriber('validator',
callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub.activate_subscription(subscription_id)
self.addCleanup(self.pubsub.deactivate_subscription, subscription_id)
self.dataset_modified = Event()
def cb2(*args, **kwargs):
self.dataset_modified.set()
# TODO: event isn't using the ExternalDataset, but a different ID for a Dataset
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb2,
origin=self.dataset_id)
es.start()
self.addCleanup(es.stop)
def test_serialize_compatability(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('ctd extended', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('ctd1', 'xp1', stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
sub_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
verified = Event()
def verifier(msg, route, stream_id):
for k,v in msg.record_dictionary.iteritems():
if v is not None:
self.assertIsInstance(v, np.ndarray)
rdt = RecordDictionaryTool.load_from_granule(msg)
for k,v in rdt.iteritems():
self.assertIsInstance(rdt[k], np.ndarray)
self.assertIsInstance(v, np.ndarray)
verified.set()
subscriber = StandaloneStreamSubscriber('sub1', callback=verifier)
subscriber.start()
self.addCleanup(subscriber.stop)
publisher = StandaloneStreamPublisher(stream_id,route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
ph.fill_rdt(rdt,10)
publisher.publish(rdt.to_granule())
self.assertTrue(verified.wait(60))
def test_data_product_subscription(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='ctd parsed')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(data_product=dp, stream_definition_id=stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
subscription_id = self.pubsub_management.create_subscription('validator', data_product_ids=[data_product_id])
self.addCleanup(self.pubsub_management.delete_subscription, subscription_id)
validated = Event()
def validation(msg, route, stream_id):
validated.set()
stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
dp_stream_id = stream_ids.pop()
validator = StandaloneStreamSubscriber('validator', callback=validation)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_management.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, subscription_id)
route = self.pubsub_management.read_stream_route(dp_stream_id)
publisher = StandaloneStreamPublisher(dp_stream_id, route)
publisher.publish('hi')
self.assertTrue(validated.wait(10))
def create_output_data_product(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject( RT.DataProduct,
name='data_process1_data_product',
description='output of add array func',
temporal_domain = self.time_dom.dump(),
spatial_domain = self.spatial_dom.dump())
dp1_func_output_dp_id = self.dataproductclient.create_data_product(dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product, dp1_func_output_dp_id)
# retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id, PRED.hasStream, None, True)
self._output_stream_ids.append(stream_ids[0])
subscription_id = self.pubsub_client.create_subscription('validator', data_product_ids=[dp1_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription, subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s', stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
self.granule_verified.set()
validator = StandaloneStreamSubscriber('validator', callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, subscription_id)
return dp1_func_output_dp_id
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._raw_samples_received = []
self._async_sample_result = AsyncResult()
self._async_raw_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received parsed data on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self._samples_received.append(message)
from pyon.util.containers import create_unique_identifier
stream_name = 'ctdpf_parsed'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" %
stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _start_subscriber_to_transform(self,
name_of_transform='',
stream_id=''):
ar = gevent.event.AsyncResult()
def subscriber(m, r, s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub_%s' %
name_of_transform,
callback=subscriber)
# Note that this running the below line creates an exchange since none of that name exists before
sub_id = self.pubsub.create_subscription(
'subscriber_to_transform_%s' % name_of_transform,
stream_ids=[stream_id],
exchange_name='sub_%s' % name_of_transform)
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
return ar
def test_event_in_stream_out_transform(self):
"""
Test the event-in/stream-out transform
"""
stream_id, _ = self.pubsub.create_stream('test_stream', exchange_point='science_data')
self.exchange_cleanup.append('science_data')
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='EventToStreamTransform',
description='For testing an event-in/stream-out transform')
process_definition.executable['module']= 'ion.processes.data.transforms.event_in_stream_out_transform'
process_definition.executable['class'] = 'EventToStreamTransform'
proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = 'test_queue'
config.process.exchange_point = 'science_data'
config.process.publish_streams.output = stream_id
config.process.event_type = 'ExampleDetectableEvent'
config.process.variables = ['voltage', 'temperature' ]
# Schedule the process
pid = self.process_dispatcher.schedule_process(process_definition_id=proc_def_id, configuration=config)
self.addCleanup(self.process_dispatcher.cancel_process,pid)
#---------------------------------------------------------------------------------------------
# Create a subscriber for testing
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber_callback(m, r, s):
ar_cond.set(m)
sub = StandaloneStreamSubscriber('sub', subscriber_callback)
self.addCleanup(sub.stop)
sub_id = self.pubsub.create_subscription('subscription_cond',
stream_ids=[stream_id],
exchange_name='sub')
self.pubsub.activate_subscription(sub_id)
self.queue_cleanup.append(sub.xn.queue)
sub.start()
gevent.sleep(4)
#---------------------------------------------------------------------------------------------
# Publish an event. The transform has been configured to receive this event
#---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ExampleDetectableEvent")
event_publisher.publish_event(origin = 'fake_origin', voltage = '5', temperature = '273')
# Assert that the transform processed the event and published data on the output stream
result_cond = ar_cond.get(timeout=10)
self.assertTrue(result_cond)
def test_granule(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':"GA03FLMA-RI001-13-CTDMOG999"})
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
self.addCleanup(self.pubsub_management.delete_stream_definition,stream_def_id)
stream_id, route = self.pubsub_management.create_stream('ctd_stream', 'xp1', stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream,stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
subscriber = StandaloneStreamSubscriber('sub', self.verify_incoming)
subscriber.start()
self.addCleanup(subscriber.stop)
subscription_id = self.pubsub_management.create_subscription('sub', stream_ids=[stream_id])
self.pubsub_management.activate_subscription(subscription_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.random.randn(10) * 10 + 30
rdt['pressure'] = [20] * 10
self.assertEquals(set(pdict.keys()), set(rdt.fields))
self.assertEquals(pdict.temporal_parameter_name, rdt.temporal_parameter)
self.assertEquals(rdt._stream_config['reference_designator'],"GA03FLMA-RI001-13-CTDMOG999")
self.rdt = rdt
self.data_producer_id = 'data_producer'
self.provider_metadata_update = {1:1}
publisher.publish(rdt.to_granule(data_producer_id='data_producer', provider_metadata_update={1:1}))
self.assertTrue(self.event.wait(10))
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.array([None,None,None])
self.assertTrue(rdt['time'] is None)
rdt['time'] = np.array([None, 1, 2])
self.assertEquals(rdt['time'][0], rdt.fill_value('time'))
stream_def_obj = self.pubsub_management.read_stream_definition(stream_def_id)
rdt = RecordDictionaryTool(stream_definition=stream_def_obj)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
granule = rdt.to_granule()
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
np.testing.assert_array_equal(rdt['temp'], np.arange(20))
def _start_data_subscribers(self, count, raw_count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._raw_samples_received = []
self._async_sample_result = AsyncResult()
self._async_raw_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received parsed data on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self._samples_received.append(message)
if len(self._samples_received) == count:
self._async_sample_result.set()
def recv_raw_data(message, stream_route, stream_id):
log.info('Received raw data on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self._raw_samples_received.append(message)
if len(self._raw_samples_received) == raw_count:
self._async_raw_sample_result.set()
from pyon.util.containers import create_unique_identifier
stream_name = 'parsed'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" % stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id],
timeout=120.2)
pubsub_client.activate_subscription(sub_id, timeout=120.3)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
stream_name = 'raw'
parsed_config = self._stream_config[stream_name]
stream_id = parsed_config['stream_id']
exchange_name = create_unique_identifier("%s_queue" % stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_raw_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id],
timeout=120.4)
pubsub_client.activate_subscription(sub_id, timeout=120.5)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def create_output_data_products(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject(RT.DataProduct,
name='data_process1_data_product',
description='output of add array func')
dp1_func_output_dp_id = self.dataproductclient.create_data_product(
dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp1_func_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_one_id = stream_ids[0]
dp2_func_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp2_stream', parameter_dictionary_id=self.parameter_dict_id)
dp2_func_output_dp_obj = IonObject(
RT.DataProduct,
name='data_process2_data_product',
description='output of add array func')
dp2_func_output_dp_id = self.dataproductclient.create_data_product(
dp2_func_output_dp_obj, dp2_func_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp2_func_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp2_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_two_id = stream_ids[0]
subscription_id = self.pubsub_client.create_subscription(
'validator',
data_product_ids=[dp1_func_output_dp_id, dp2_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription,
subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s',
stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
validator = StandaloneStreamSubscriber('validator',
callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
subscription_id)
return dp1_func_output_dp_id, dp2_func_output_dp_id
def _start_data_subscribers(self, count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
#self._async_data_result = AsyncResult()
strXterm = "xterm -T InstrumentScienceData -sb -rightbar "
pOpenString = strXterm + " -e tail -f " + PIPE_PATH
subprocess.Popen([
'xterm', '-T', 'InstrumentScienceData', '-e', 'tail', '-f',
PIPE_PATH
])
#subprocess.Popen(pOpenString)
#self.pipeData = open(PIPE_PATH, "w", 1)
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
print 'Received message on ' + str(stream_id) + ' (' + str(
stream_route.exchange_point) + ',' + str(
stream_route.routing_key) + ')'
log.info('Received message on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self.pipeData = open(PIPE_PATH, "w", 1)
self.pipeData.write(str(message))
self.pipeData.flush()
self.pipeData.close()
self._samples_received.append(message)
#if len(self._samples_received) == count:
#self._async_data_result.set()
for (stream_name, stream_config) in self._stream_config.iteritems():
stream_id = stream_config['stream_id']
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
print 'stream_id: %s' % stream_id
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def test_full_pubsub(self):
self.sub1_sat = Event()
self.sub2_sat = Event()
def subscriber1(m, r, s):
self.sub1_sat.set()
def subscriber2(m, r, s):
self.sub2_sat.set()
sub1 = StandaloneStreamSubscriber("sub1", subscriber1)
self.queue_cleanup.append(sub1.xn.queue)
sub1.start()
sub2 = StandaloneStreamSubscriber("sub2", subscriber2)
self.queue_cleanup.append(sub2.xn.queue)
sub2.start()
log_topic = self.pubsub_management.create_topic("instrument_logs", exchange_point="instruments")
science_topic = self.pubsub_management.create_topic("science_data", exchange_point="instruments")
events_topic = self.pubsub_management.create_topic("notifications", exchange_point="events")
log_stream, route = self.pubsub_management.create_stream(
"instrument1-logs", topic_ids=[log_topic], exchange_point="instruments"
)
ctd_stream, route = self.pubsub_management.create_stream(
"instrument1-ctd", topic_ids=[science_topic], exchange_point="instruments"
)
event_stream, route = self.pubsub_management.create_stream(
"notifications", topic_ids=[events_topic], exchange_point="events"
)
raw_stream, route = self.pubsub_management.create_stream("temp", exchange_point="global.data")
self.exchange_cleanup.extend(["instruments", "events", "global.data"])
subscription1 = self.pubsub_management.create_subscription(
"subscription1", stream_ids=[log_stream, event_stream], exchange_name="sub1"
)
subscription2 = self.pubsub_management.create_subscription(
"subscription2", exchange_points=["global.data"], stream_ids=[ctd_stream], exchange_name="sub2"
)
self.pubsub_management.activate_subscription(subscription1)
self.pubsub_management.activate_subscription(subscription2)
self.publish_on_stream(log_stream, 1)
self.assertTrue(self.sub1_sat.wait(4))
self.assertFalse(self.sub2_sat.is_set())
self.publish_on_stream(raw_stream, 1)
self.assertTrue(self.sub1_sat.wait(4))
sub1.stop()
sub2.stop()
def test_move_activated_subscription(self):
stream_id, route = self.pubsub_management.create_stream(
name='test_stream', exchange_point='test_xp')
#--------------------------------------------------------------------------------
# Test moving after activate
#--------------------------------------------------------------------------------
subscription_id = self.pubsub_management.create_subscription(
'first_queue', stream_ids=[stream_id])
self.pubsub_management.activate_subscription(subscription_id)
xn_ids, _ = self.resource_registry.find_resources(
restype=RT.ExchangeName, name='first_queue', id_only=True)
subjects, _ = self.resource_registry.find_subjects(
object=subscription_id,
predicate=PRED.hasSubscription,
id_only=True)
self.assertEquals(xn_ids[0], subjects[0])
self.verified = Event()
def verify(m, r, s):
self.assertEquals(m, 'verified')
self.verified.set()
subscriber = StandaloneStreamSubscriber('second_queue', verify)
subscriber.start()
self.pubsub_management.move_subscription(subscription_id,
exchange_name='second_queue')
xn_ids, _ = self.resource_registry.find_resources(
restype=RT.ExchangeName, name='second_queue', id_only=True)
subjects, _ = self.resource_registry.find_subjects(
object=subscription_id,
predicate=PRED.hasSubscription,
id_only=True)
self.assertEquals(len(subjects), 1)
self.assertEquals(subjects[0], xn_ids[0])
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish('verified')
self.assertTrue(self.verified.wait(2))
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
self.pubsub_management.delete_stream(stream_id)
def _start_data_subscriber(self, config, callback):
"""
Setup and start a data subscriber
"""
exchange_point = config['exchange_point']
stream_id = config['stream_id']
sub = StandaloneStreamSubscriber(exchange_point, callback)
sub.start()
self._data_subscribers.append(sub)
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
sub_id = pubsub_client.create_subscription(name=exchange_point, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def start_output_stream_and_listen(self, ctd_stream_id, data_product_stream_ids, message_count_per_stream=10):
assertions = self.assertTrue
exchange_name = 'workflow_test'
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
name = 'test workflow transformations',
exchange_name = exchange_name,
stream_ids = data_product_stream_ids
)
result = gevent.event.AsyncResult()
results = []
message_count = len(data_product_stream_ids) * message_count_per_stream
def message_received(message, stream_route, stream_id):
# Heads
results.append(message)
if len(results) >= message_count: #Only wait for so many messages - per stream
result.set(True)
subscriber = StandaloneStreamSubscriber(exchange_name='workflow_test', callback=message_received)
subscriber.xn.purge()
self.addCleanup(subscriber.xn.delete)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
#Start the input stream process
if ctd_stream_id is not None:
ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id)
# Assert that we have received data
assertions(result.get(timeout=30))
# stop the flow parse the messages...
if ctd_stream_id is not None:
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def _start_data_subscriber(self, config, callback):
"""
Setup and start a data subscriber
"""
exchange_point = config['exchange_point']
stream_id = config['stream_id']
sub = StandaloneStreamSubscriber(exchange_point, callback)
sub.start()
self._data_subscribers.append(sub)
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
sub_id = pubsub_client.create_subscription(name=exchange_point,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _start_data_subscribers(self):
"""
"""
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
#
# TODO retrieve appropriate stream definitions; for the moment, using
# adhoc_get_stream_names
#
# A callback for processing subscribed-to data.
def consume_data(message, stream_route, stream_id):
log.info('Subscriber received data message: %s.' % str(message))
self._samples_received.append(message)
self._async_data_result.set()
for stream_name in adhoc_get_stream_names():
log.info('creating stream %r ...', stream_name)
# TODO use appropriate exchange_point
stream_id, stream_route = self._pubsub_client.create_stream(
name=stream_name, exchange_point='science_data')
log.info('create_stream(%r): stream_id=%r, stream_route=%s',
stream_name, stream_id, str(stream_route))
pdict = adhoc_get_parameter_dictionary(stream_name)
stream_config = dict(stream_route=stream_route.routing_key,
stream_id=stream_id,
parameter_dictionary=pdict.dump())
self._stream_config[stream_name] = stream_config
log.info('_stream_config[%r]= %r', stream_name, stream_config)
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self._pubsub_client.create_subscription(
name=exchange_name, stream_ids=[stream_id])
self._pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id
def test_transform_prime_no_available_fields(self):
available_fields_in = []
available_fields_out = []
exchange_pt1 = 'xp1'
exchange_pt2 = 'xp2'
stream_id_in,stream_id_out,stream_route_in,stream_route_out,stream_def_in_id,stream_def_out_id = self._setup_streams(exchange_pt1, exchange_pt2, available_fields_in, available_fields_out)
#launch transform
config = {'process':{'routes':{(stream_id_in, stream_id_out):None},'queue_name':exchange_pt1, 'publish_streams':{str(stream_id_out):stream_id_out}, 'process_type':'stream_process'}}
pid = self.container.spawn_process('transform_stream','ion.processes.data.transforms.transform_prime','TransformPrime',config)
#create publish
publisher = StandaloneStreamPublisher(stream_id_in, stream_route_in)
self.container.proc_manager.procs[pid].subscriber.xn.bind(stream_route_in.routing_key, publisher.xp)
#data
rdt_in = RecordDictionaryTool(stream_definition_id=stream_def_in_id)
dt = 20
rdt_in['time'] = np.arange(dt)
rdt_in['lat'] = [40.992469] * dt
rdt_in['lon'] = [-71.727069] * dt
rdt_in['TEMPWAT_L0'] = self._get_param_vals('TEMPWAT_L0', slice(None), (dt,))
rdt_in['CONDWAT_L0'] = self._get_param_vals('CONDWAT_L0', slice(None), (dt,))
rdt_in['PRESWAT_L0'] = self._get_param_vals('PRESWAT_L0', slice(None), (dt,))
msg = rdt_in.to_granule()
#publish granule to transform and have transform publish it to subsciber
#validate transformed data
e = gevent.event.Event()
def cb(msg, sr, sid):
self.assertEqual(sid, stream_id_out)
rdt_out = RecordDictionaryTool.load_from_granule(msg)
self.assertEquals(set([k for k,v in rdt_out.iteritems()]), set(available_fields_out))
for k,v in rdt_out.iteritems():
self.assertEquals(rdt_out[k], None)
e.set()
sub = StandaloneStreamSubscriber('stream_subscriber', cb)
sub.xn.bind(stream_route_out.routing_key, getattr(self.container.proc_manager.procs[pid], stream_id_out).xp)
self.addCleanup(sub.stop)
sub.start()
#publish msg to transform
publisher.publish(msg)
#wait to receive msg
self.assertTrue(e.wait(4))
def _start_data_subscribers(self, count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
#self._async_data_result = AsyncResult()
strXterm = "xterm -T InstrumentScienceData -sb -rightbar "
pOpenString = strXterm + " -e tail -f " + PIPE_PATH
subprocess.Popen(['xterm', '-T', 'InstrumentScienceData', '-e', 'tail', '-f', PIPE_PATH])
#subprocess.Popen(pOpenString)
#self.pipeData = open(PIPE_PATH, "w", 1)
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
print 'Received message on ' + str(stream_id) + ' (' + str(stream_route.exchange_point) + ',' + str(stream_route.routing_key) + ')'
log.info('Received message on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self.pipeData = open(PIPE_PATH, "w", 1)
self.pipeData.write(str(message))
self.pipeData.flush()
self.pipeData.close()
self._samples_received.append(message)
#if len(self._samples_received) == count:
#self._async_data_result.set()
for (stream_name, stream_config) in self._stream_config.iteritems():
stream_id = stream_config['stream_id']
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
print 'stream_id: %s' % stream_id
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def test_full_pubsub(self):
self.sub1_sat = Event()
self.sub2_sat = Event()
def subscriber1(m,r,s):
self.sub1_sat.set()
def subscriber2(m,r,s):
self.sub2_sat.set()
sub1 = StandaloneStreamSubscriber('sub1', subscriber1)
self.queue_cleanup.append(sub1.xn.queue)
sub1.start()
sub2 = StandaloneStreamSubscriber('sub2', subscriber2)
self.queue_cleanup.append(sub2.xn.queue)
sub2.start()
log_topic = self.pubsub_management.create_topic('instrument_logs', exchange_point='instruments')
science_topic = self.pubsub_management.create_topic('science_data', exchange_point='instruments')
events_topic = self.pubsub_management.create_topic('notifications', exchange_point='events')
log_stream, route = self.pubsub_management.create_stream('instrument1-logs', topic_ids=[log_topic], exchange_point='instruments')
ctd_stream, route = self.pubsub_management.create_stream('instrument1-ctd', topic_ids=[science_topic], exchange_point='instruments')
event_stream, route = self.pubsub_management.create_stream('notifications', topic_ids=[events_topic], exchange_point='events')
raw_stream, route = self.pubsub_management.create_stream('temp', exchange_point='global.data')
self.exchange_cleanup.extend(['instruments','events','global.data'])
subscription1 = self.pubsub_management.create_subscription('subscription1', stream_ids=[log_stream,event_stream], exchange_name='sub1')
subscription2 = self.pubsub_management.create_subscription('subscription2', exchange_points=['global.data'], stream_ids=[ctd_stream], exchange_name='sub2')
self.pubsub_management.activate_subscription(subscription1)
self.pubsub_management.activate_subscription(subscription2)
self.publish_on_stream(log_stream, 1)
self.assertTrue(self.sub1_sat.wait(4))
self.assertFalse(self.sub2_sat.is_set())
self.publish_on_stream(raw_stream,1)
self.assertTrue(self.sub1_sat.wait(4))
sub1.stop()
sub2.stop()
def _start_data_subscribers(self):
"""
"""
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
#
# TODO retrieve appropriate stream definitions; for the moment, using
# adhoc_get_stream_names
#
# A callback for processing subscribed-to data.
def consume_data(message, stream_route, stream_id):
log.info('Subscriber received data message: %s.' % str(message))
self._samples_received.append(message)
self._async_data_result.set()
for stream_name in adhoc_get_stream_names():
log.info('creating stream %r ...', stream_name)
# TODO use appropriate exchange_point
stream_id, stream_route = self._pubsub_client.create_stream(
name=stream_name, exchange_point='science_data')
log.info('create_stream(%r): stream_id=%r, stream_route=%s',
stream_name, stream_id, str(stream_route))
pdict = adhoc_get_parameter_dictionary(stream_name)
stream_config = dict(stream_route=stream_route.routing_key,
stream_id=stream_id,
parameter_dictionary=pdict.dump())
self._stream_config[stream_name] = stream_config
log.info('_stream_config[%r]= %r', stream_name, stream_config)
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self._pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
self._pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id
def setup_subscriber(self, data_product_id, callback):
stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
self.assertTrue(len(stream_ids))
stream_id = stream_ids.pop()
sub_id = self.pubsub_management.create_subscription('validator_%s'%self.validators, stream_ids=[stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
subscriber = StandaloneStreamSubscriber('validator_%s' % self.validators, callback=callback)
subscriber.start()
self.addCleanup(subscriber.stop)
self.validators+=1
return subscriber
def test_move_activated_subscription(self):
stream_id, route = self.pubsub_management.create_stream(name="test_stream", exchange_point="test_xp")
# --------------------------------------------------------------------------------
# Test moving after activate
# --------------------------------------------------------------------------------
subscription_id = self.pubsub_management.create_subscription("first_queue", stream_ids=[stream_id])
self.pubsub_management.activate_subscription(subscription_id)
xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="first_queue", id_only=True)
subjects, _ = self.resource_registry.find_subjects(
object=subscription_id, predicate=PRED.hasSubscription, id_only=True
)
self.assertEquals(xn_ids[0], subjects[0])
self.verified = Event()
def verify(m, r, s):
self.assertEquals(m, "verified")
self.verified.set()
subscriber = StandaloneStreamSubscriber("second_queue", verify)
subscriber.start()
self.pubsub_management.move_subscription(subscription_id, exchange_name="second_queue")
xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="second_queue", id_only=True)
subjects, _ = self.resource_registry.find_subjects(
object=subscription_id, predicate=PRED.hasSubscription, id_only=True
)
self.assertEquals(len(subjects), 1)
self.assertEquals(subjects[0], xn_ids[0])
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish("verified")
self.assertTrue(self.verified.wait(2))
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
self.pubsub_management.delete_stream(stream_id)
def _start_data_subscribers(self, count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._async_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received message on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
#print '######################## stream message:'
#print str(message)
self._samples_received.append(message)
if len(self._samples_received) == count:
self._async_sample_result.set()
for (stream_name, stream_config) in self._stream_config.iteritems():
if stream_name == 'parsed':
# Create subscription for parsed stream only.
stream_id = stream_config['stream_id']
from pyon.util.containers import create_unique_identifier
# exchange_name = '%s_queue' % stream_name
exchange_name = create_unique_identifier(
"%s_queue" % stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
print 'stream_id: %s' % stream_id
sub_id = pubsub_client.create_subscription(
name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self.data_subscribers = {}
self.samples_received = {}
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received message on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
name = None
for (stream_name, stream_config) in self.stream_config.iteritems():
if stream_route.routing_key == stream_config['routing_key']:
log.debug("NAME: %s %s == %s" %
(stream_name, stream_route.routing_key,
stream_config['routing_key']))
name = stream_name
if (not self.samples_received.get(stream_name)):
self.samples_received[stream_name] = []
self.samples_received[stream_name].append(message)
log.debug("Add message to stream '%s' value: %s" %
(stream_name, message))
for (stream_name, stream_config) in self.stream_config.iteritems():
stream_id = stream_config['stream_id']
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self.data_subscribers[stream_name] = sub
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def test_activation_and_deactivation(self):
stream_id, route = self.pubsub_management.create_stream('stream1','xp1')
subscription_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id])
self.check1 = Event()
def verifier(m,r,s):
self.check1.set()
subscriber = StandaloneStreamSubscriber('sub1',verifier)
subscriber.start()
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish('should not receive')
self.assertFalse(self.check1.wait(0.25))
self.pubsub_management.activate_subscription(subscription_id)
publisher.publish('should receive')
self.assertTrue(self.check1.wait(2))
self.check1.clear()
self.assertFalse(self.check1.is_set())
self.pubsub_management.deactivate_subscription(subscription_id)
publisher.publish('should not receive')
self.assertFalse(self.check1.wait(0.5))
self.pubsub_management.activate_subscription(subscription_id)
publisher.publish('should receive')
self.assertTrue(self.check1.wait(2))
subscriber.stop()
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
self.pubsub_management.delete_stream(stream_id)
def start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self.data_subscribers = {}
self.samples_received = {}
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info("Received message on %s (%s,%s)", stream_id, stream_route.exchange_point, stream_route.routing_key)
name = None
for (stream_name, stream_config) in self.stream_config.iteritems():
if stream_route.routing_key == stream_config["routing_key"]:
log.debug(
"NAME: %s %s == %s" % (stream_name, stream_route.routing_key, stream_config["routing_key"])
)
name = stream_name
if not self.samples_received.get(stream_name):
self.samples_received[stream_name] = []
self.samples_received[stream_name].append(message)
log.debug("Add message to stream '%s' value: %s" % (stream_name, message))
for (stream_name, stream_config) in self.stream_config.iteritems():
stream_id = stream_config["stream_id"]
# Create subscriptions for each stream.
exchange_name = "%s_queue" % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self.data_subscribers[stream_name] = sub
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = (
sub_id
) # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _start_subscriber_to_transform(self, name_of_transform = '', stream_id = ''):
ar = gevent.event.AsyncResult()
def subscriber(m,r,s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub_%s' % name_of_transform, callback=subscriber)
# Note that this running the below line creates an exchange since none of that name exists before
sub_id = self.pubsub.create_subscription('subscriber_to_transform_%s' % name_of_transform,
stream_ids=[stream_id],
exchange_name='sub_%s' % name_of_transform)
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
return ar
def _start_data_subscribers(self, count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
self._async_sample_result = AsyncResult()
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
log.info('Received message on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
#print '######################## stream message:'
#print str(message)
self._samples_received.append(message)
if len(self._samples_received) == count:
self._async_sample_result.set()
for (stream_name, stream_config) in self._stream_config.iteritems():
if stream_name == 'parsed':
# Create subscription for parsed stream only.
stream_id = stream_config['stream_id']
from pyon.util.containers import create_unique_identifier
# exchange_name = '%s_queue' % stream_name
exchange_name = create_unique_identifier("%s_queue" %
stream_name)
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
print 'stream_id: %s' % stream_id
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def test_demux(self):
self.stream0, self.route0 = self.pubsub_client.create_stream('stream0', exchange_point='test')
self.stream1, self.route1 = self.pubsub_client.create_stream('stream1', exchange_point='main_data')
self.stream2, self.route2 = self.pubsub_client.create_stream('stream2', exchange_point='alt_data')
self.r_stream1 = gevent.event.Event()
self.r_stream2 = gevent.event.Event()
def process(msg, stream_route, stream_id):
if stream_id == self.stream1:
self.r_stream1.set()
elif stream_id == self.stream2:
self.r_stream2.set()
self.container.spawn_process('demuxer', 'ion.processes.data.transforms.mux', 'DemuxTransform', {'process':{'out_streams':[self.stream1, self.stream2]}}, 'demuxer_pid')
self.queue_cleanup.append('demuxer_pid')
sub1 = StandaloneStreamSubscriber('sub1', process)
sub2 = StandaloneStreamSubscriber('sub2', process)
sub1.xn.bind(self.route1.routing_key, self.container.ex_manager.create_xp('main_data'))
sub2.xn.bind(self.route2.routing_key, self.container.ex_manager.create_xp('alt_data'))
sub1.start()
sub2.start()
self.queue_cleanup.append(sub1.xn)
self.queue_cleanup.append(sub2.xn)
xn = self.container.ex_manager.create_xn_queue('demuxer_pid')
xn.bind(self.route0.routing_key, self.container.ex_manager.create_xp(self.route0.exchange_point))
domino = StandaloneStreamPublisher(self.stream0, self.route0)
domino.publish('test')
self.assertTrue(self.r_stream1.wait(2))
self.assertTrue(self.r_stream2.wait(2))
self.container.proc_manager.terminate_process('demuxer_pid')
sub1.stop()
sub2.stop()
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name,
stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def start_output_stream_and_listen(self,
ctd_stream_id,
data_product_stream_ids,
message_count_per_stream=10):
assertions = self.assertTrue
exchange_name = 'workflow_test'
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
name='test workflow transformations',
exchange_name=exchange_name,
stream_ids=data_product_stream_ids)
result = gevent.event.AsyncResult()
results = []
message_count = len(data_product_stream_ids) * message_count_per_stream
def message_received(message, stream_route, stream_id):
# Heads
results.append(message)
if len(
results
) >= message_count: #Only wait for so many messages - per stream
result.set(True)
subscriber = StandaloneStreamSubscriber(exchange_name='workflow_test',
callback=message_received)
subscriber.xn.purge()
self.addCleanup(subscriber.xn.delete)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(
subscription_id=salinity_subscription_id)
#Start the input stream process
if ctd_stream_id is not None:
ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id)
# Assert that we have received data
assertions(result.get(timeout=30))
# stop the flow parse the messages...
if ctd_stream_id is not None:
self.process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
self.pubsubclient.deactivate_subscription(
subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def test_granule(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd', parameter_dictionary_id=pdict_id)
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
stream_id, route = self.pubsub_management.create_stream('ctd_stream', 'xp1', stream_definition_id=stream_def_id)
self.xps.append('xp1')
publisher = StandaloneStreamPublisher(stream_id, route)
subscriber = StandaloneStreamSubscriber('sub', self.verify_incoming)
subscriber.start()
subscription_id = self.pubsub_management.create_subscription('sub', stream_ids=[stream_id])
self.xns.append('sub')
self.pubsub_management.activate_subscription(subscription_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.random.randn(10) * 10 + 30
rdt['pressure'] = [20] * 10
self.assertEquals(set(pdict.keys()), set(rdt.fields))
self.assertEquals(pdict.temporal_parameter_name, rdt.temporal_parameter)
self.rdt = rdt
self.data_producer_id = 'data_producer'
self.provider_metadata_update = {1:1}
publisher.publish(rdt.to_granule(data_producer_id='data_producer', provider_metadata_update={1:1}))
self.assertTrue(self.event.wait(10))
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
def do_listen_for_incoming(self):
subscription_id = self.pubsub.create_subscription('validator', data_product_ids=[self.data_product._id])
self.addCleanup(self.pubsub.delete_subscription, subscription_id)
self.granule_capture = []
self.granule_count = 0
def on_granule(msg, route, stream_id):
self.granule_count += 1
if self.granule_count<5:
self.granule_capture.append(msg)
validator = StandaloneStreamSubscriber('validator', callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub.activate_subscription(subscription_id)
self.addCleanup(self.pubsub.deactivate_subscription, subscription_id)
self.dataset_modified = Event()
def cb2(*args, **kwargs):
self.dataset_modified.set()
# TODO: event isn't using the ExternalDataset, but a different ID for a Dataset
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb2, origin=self.device._id)
es.start()
self.addCleanup(es.stop)
def test_activation_and_deactivation(self):
stream_id, route = self.pubsub_management.create_stream(
'stream1', 'xp1')
subscription_id = self.pubsub_management.create_subscription(
'sub1', stream_ids=[stream_id])
self.check1 = Event()
def verifier(m, r, s):
self.check1.set()
subscriber = StandaloneStreamSubscriber('sub1', verifier)
subscriber.start()
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish('should not receive')
self.assertFalse(self.check1.wait(0.25))
self.pubsub_management.activate_subscription(subscription_id)
publisher.publish('should receive')
self.assertTrue(self.check1.wait(2))
self.check1.clear()
self.assertFalse(self.check1.is_set())
self.pubsub_management.deactivate_subscription(subscription_id)
publisher.publish('should not receive')
self.assertFalse(self.check1.wait(0.5))
self.pubsub_management.activate_subscription(subscription_id)
publisher.publish('should receive')
self.assertTrue(self.check1.wait(2))
subscriber.stop()
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
self.pubsub_management.delete_stream(stream_id)
def test_ctd_L0_all(self):
'''
Test that packets are processed by the ctd_L0_all transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='ctd_L0_all',
description='For testing ctd_L0_all')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.ctd_L0_all'
process_definition.executable['class'] = 'ctd_L0_all'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('ctd_all_stream_def', parameter_dictionary_id=pdict_id)
cond_stream_id, _ = self.pubsub.create_stream('test_cond',
exchange_point='science_data',
stream_definition_id=stream_def_id)
pres_stream_id, _ = self.pubsub.create_stream('test_pres',
exchange_point='science_data',
stream_definition_id=stream_def_id)
temp_stream_id, _ = self.pubsub.create_stream('test_temp',
exchange_point='science_data',
stream_definition_id=stream_def_id)
config.process.publish_streams.conductivity = cond_stream_id
config.process.publish_streams.pressure = pres_stream_id
config.process.publish_streams.temperature = temp_stream_id
# Schedule the process
pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscribers that will receive the conductivity, temperature and pressure granules from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber1(m, r, s):
ar_cond.set(m)
sub_cond = StandaloneStreamSubscriber('sub_cond', subscriber1)
self.addCleanup(sub_cond.stop)
ar_temp = gevent.event.AsyncResult()
def subscriber2(m,r,s):
ar_temp.set(m)
sub_temp = StandaloneStreamSubscriber('sub_temp', subscriber2)
self.addCleanup(sub_temp.stop)
ar_pres = gevent.event.AsyncResult()
def subscriber3(m,r,s):
ar_pres.set(m)
sub_pres = StandaloneStreamSubscriber('sub_pres', subscriber3)
self.addCleanup(sub_pres.stop)
sub_cond_id= self.pubsub.create_subscription('subscription_cond',
stream_ids=[cond_stream_id],
exchange_name='sub_cond')
sub_temp_id = self.pubsub.create_subscription('subscription_temp',
stream_ids=[temp_stream_id],
exchange_name='sub_temp')
sub_pres_id = self.pubsub.create_subscription('subscription_pres',
stream_ids=[pres_stream_id],
exchange_name='sub_pres')
self.pubsub.activate_subscription(sub_cond_id)
self.pubsub.activate_subscription(sub_temp_id)
self.pubsub.activate_subscription(sub_pres_id)
self.queue_cleanup.append(sub_cond.xn.queue)
self.queue_cleanup.append(sub_temp.xn.queue)
self.queue_cleanup.append(sub_pres.xn.queue)
sub_cond.start()
sub_temp.start()
sub_pres.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result_cond = ar_cond.get(timeout=10)
result_temp = ar_temp.get(timeout=10)
result_pres = ar_pres.get(timeout=10)
out_dict = {}
out_dict['c'] = RecordDictionaryTool.load_from_granule(result_cond)['conductivity']
out_dict['t'] = RecordDictionaryTool.load_from_granule(result_temp)['temp']
out_dict['p'] = RecordDictionaryTool.load_from_granule(result_pres)['pressure']
# Check that the transform algorithm was successfully executed
self.check_granule_splitting(publish_granule, out_dict)
def test_dm_end_2_end(self):
#--------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
#--------------------------------------------------------------------------------
self.event.clear()
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
#--------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
#--------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
#--------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
#--------------------------------------------------------------------------------
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id,40)
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:])
self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all())
#--------------------------------------------------------------------------------
# Now to try the streamed approach
#--------------------------------------------------------------------------------
replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id)
log.info('Process ID: %s', process_id)
replay_client = ReplayClient(process_id)
#--------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
#--------------------------------------------------------------------------------
sub_id = self.pubsub_management.create_subscription(self.exchange_space_name,stream_ids=[replay_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
subscriber.start()
self.addCleanup(subscriber.stop)
self.data_retriever.start_replay_agent(self.replay_id)
self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched')
replay_client.start_replay()
self.assertTrue(self.event.wait(10))
self.data_retriever.cancel_replay_agent(self.replay_id)
#--------------------------------------------------------------------------------
# Test the slicing capabilities
#--------------------------------------------------------------------------------
granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)})
rdt = RecordDictionaryTool.load_from_granule(granule)
b = rdt['time'] == np.arange(5)
self.assertTrue(b.all() if not isinstance(b,bool) else b)
def test_event_triggered_transform_A(self):
"""
Test that packets are processed by the event triggered transform
"""
# ---------------------------------------------------------------------------------------------
# Launch a ctd transform
# ---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name="EventTriggeredTransform_A", description="For testing EventTriggeredTransform_A"
)
process_definition.executable["module"] = "ion.processes.data.transforms.event_triggered_transform"
process_definition.executable["class"] = "EventTriggeredTransform_A"
event_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition
)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
stream_def_id = self.pubsub.create_stream_definition("cond_stream_def", parameter_dictionary_id=pdict_id)
cond_stream_id, _ = self.pubsub.create_stream(
"test_conductivity", exchange_point="science_data", stream_definition_id=stream_def_id
)
config.process.publish_streams.conductivity = cond_stream_id
config.process.event_type = "ResourceLifecycleEvent"
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=event_transform_proc_def_id, configuration=config
)
# ---------------------------------------------------------------------------------------------
# Publish an event to wake up the event triggered transform
# ---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ResourceLifecycleEvent")
event_publisher.publish_event(origin="fake_origin")
# ---------------------------------------------------------------------------------------------
# Create subscribers that will receive the conductivity, temperature and pressure granules from
# the ctd transform
# ---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber1(m, r, s):
ar_cond.set(m)
sub_event_transform = StandaloneStreamSubscriber("sub_event_transform", subscriber1)
self.addCleanup(sub_event_transform.stop)
sub_event_transform_id = self.pubsub.create_subscription(
"subscription_cond", stream_ids=[cond_stream_id], exchange_name="sub_event_transform"
)
self.pubsub.activate_subscription(sub_event_transform_id)
self.queue_cleanup.append(sub_event_transform.xn.queue)
sub_event_transform.start()
# ------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
# ------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = "stream_id.stream"
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind("stream_id.stream", xp)
pub = StandaloneStreamPublisher("stream_id", stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
# ------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
# ------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result_cond = ar_cond.get(timeout=10)
self.assertTrue(isinstance(result_cond, Granule))
rdt = RecordDictionaryTool.load_from_granule(result_cond)
self.assertTrue(rdt.__contains__("conductivity"))
self.check_cond_algorithm_execution(publish_granule, result_cond)
def setup_validator(self, validator):
listener = StandaloneStreamSubscriber('listener', validator)
listener.start()
self.addCleanup(listener.stop)
def test_ctdbp_L0_all(self):
"""
Test packets processed by the ctdbp_L0_all transform
"""
#----------- Data Process Definition --------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='CTDBP_L0_all',
description=
'Take parsed stream and put the C, T and P into three separate L0 streams.',
module='ion.processes.data.transforms.ctdbp.ctdbp_L0',
class_name='CTDBP_L0_all')
dprocdef_id = self.data_process_management.create_data_process_definition(
dpd_obj)
self.addCleanup(
self.data_process_management.delete_data_process_definition,
dprocdef_id)
log.debug("created data process definition: id = %s", dprocdef_id)
#----------- Data Products --------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_param_dict = self._create_input_param_dict_for_test(
parameter_dict_name='fictitious_ctdp_param_dict')
# Get the stream definition for the stream using the parameter dictionary
# input_param_dict = self.dataset_management.read_parameter_dictionary_by_name('ctdbp_cdef_sample', id_only=True)
input_stream_def_dict = self.pubsub.create_stream_definition(
name='parsed', parameter_dictionary_id=input_param_dict)
self.addCleanup(self.pubsub.delete_stream_definition,
input_stream_def_dict)
log.debug("Got the parsed parameter dictionary: id: %s",
input_param_dict)
log.debug("Got the stream def for parsed input: %s",
input_stream_def_dict)
# Input data product
parsed_stream_dp_obj = IonObject(
RT.DataProduct,
name='parsed_stream',
description='Parsed stream input to CTBP L0 transform',
temporal_domain=tdom,
spatial_domain=sdom)
input_dp_id = self.dataproduct_management.create_data_product(
data_product=parsed_stream_dp_obj,
stream_definition_id=input_stream_def_dict)
self.addCleanup(self.dataproduct_management.delete_data_product,
input_dp_id)
# output data product
L0_stream_dp_obj = IonObject(
RT.DataProduct,
name='L0_stream',
description='L0_stream output of CTBP L0 transform',
temporal_domain=tdom,
spatial_domain=sdom)
L0_stream_dp_id = self.dataproduct_management.create_data_product(
data_product=L0_stream_dp_obj,
stream_definition_id=input_stream_def_dict)
self.addCleanup(self.dataproduct_management.delete_data_product,
L0_stream_dp_id)
# We need the key name here to be "L0_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L0_stream when the config is used to launch the data process
out_stream_ids, _ = self.resource_registry.find_objects(
L0_stream_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(out_stream_ids))
output_stream_id = out_stream_ids[0]
dproc_id = self.data_process_management.create_data_process(
data_process_definition_id=dprocdef_id,
in_data_product_ids=[input_dp_id],
out_data_product_ids=[L0_stream_dp_id],
configuration=None)
self.addCleanup(self.data_process_management.delete_data_process,
dproc_id)
log.debug("Created a data process for ctdbp_L0. id: %s", dproc_id)
# Activate the data process
self.data_process_management.activate_data_process(dproc_id)
self.addCleanup(self.data_process_management.deactivate_data_process,
dproc_id)
#----------- Find the stream that is associated with the input data product when it was created by create_data_product() --------------------------------
stream_ids, _ = self.resource_registry.find_objects(
input_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(stream_ids))
input_stream_id = stream_ids[0]
stream_route = self.pubsub.read_stream_route(input_stream_id)
log.debug("The input stream for the L0 transform: %s", input_stream_id)
#----------- Create a subscriber that will listen to the transform's output --------------------------------
ar = gevent.event.AsyncResult()
def subscriber(m, r, s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub',
callback=subscriber)
sub_id = self.pubsub.create_subscription('subscriber_to_transform',
stream_ids=[output_stream_id],
exchange_name='sub')
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
#----------- Publish on that stream so that the transform can receive it --------------------------------
pub = StandaloneStreamPublisher(input_stream_id, stream_route)
publish_granule = self._get_new_ctd_packet(
stream_definition_id=input_stream_def_dict, length=5)
pub.publish(publish_granule)
log.debug("Published the following granule: %s", publish_granule)
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the transform: %s",
granule_from_transform)
# Check that the granule published by the L0 transform has the right properties
self._check_granule_from_transform(granule_from_transform)
def test_replay_pause(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(
self.dataset_management.create_parameter_context(
'binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(
self.dataset_management.create_parameter_context(
'records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'replay_pdict',
parameter_context_ids=context_ids,
temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition(
'replay_stream', parameter_dictionary_id=pdict_id)
replay_stream, replay_route = self.pubsub_management.create_stream(
'replay', 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
scov = DatasetManagementService._get_coverage(dataset_id)
bb = CoverageCraft(scov)
bb.rdt['time'] = np.arange(100)
bb.rdt['temp'] = np.random.random(100) + 30
bb.sync_with_granule()
DatasetManagementService._persist_coverage(
dataset_id,
bb.coverage) # This invalidates it for multi-host configurations
# Set up the subscriber to verify the data
subscriber = StandaloneStreamSubscriber(
self.exchange_space_name, self.validate_granule_subscription)
xp = self.container.ex_manager.create_xp('xp1')
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
# Set up the replay agent and the client wrapper
# 1) Define the Replay (dataset and stream to publish on)
self.replay_id, process_id = self.data_retriever.define_replay(
dataset_id=dataset_id, stream_id=replay_stream)
# 2) Make a client to the interact with the process (optionall provide it a process to bind with)
replay_client = ReplayClient(process_id)
# 3) Start the agent (launch the process)
self.data_retriever.start_replay_agent(self.replay_id)
# 4) Start replaying...
replay_client.start_replay()
# Wait till we get some granules
self.assertTrue(self.event.wait(5))
# We got granules, pause the replay, clear the queue and allow the process to finish consuming
replay_client.pause_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure there's no remaining messages being consumed
self.assertFalse(self.event.wait(1))
# Resume the replay and wait until we start getting granules again
replay_client.resume_replay()
self.assertTrue(self.event.wait(5))
# Stop the replay, clear the queues
replay_client.stop_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure that it did indeed stop
self.assertFalse(self.event.wait(1))
subscriber.stop()
def test_ctd_L0_all(self):
'''
Test that packets are processed by the ctd_L0_all transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='ctd_L0_all', description='For testing ctd_L0_all')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.ctd_L0_all'
process_definition.executable['class'] = 'ctd_L0_all'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'ctd_all_stream_def', parameter_dictionary_id=pdict_id)
cond_stream_id, _ = self.pubsub.create_stream(
'test_cond',
exchange_point='science_data',
stream_definition_id=stream_def_id)
pres_stream_id, _ = self.pubsub.create_stream(
'test_pres',
exchange_point='science_data',
stream_definition_id=stream_def_id)
temp_stream_id, _ = self.pubsub.create_stream(
'test_temp',
exchange_point='science_data',
stream_definition_id=stream_def_id)
config.process.publish_streams.conductivity = cond_stream_id
config.process.publish_streams.pressure = pres_stream_id
config.process.publish_streams.temperature = temp_stream_id
# Schedule the process
pid = self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscribers that will receive the conductivity, temperature and pressure granules from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber1(m, r, s):
ar_cond.set(m)
sub_cond = StandaloneStreamSubscriber('sub_cond', subscriber1)
self.addCleanup(sub_cond.stop)
ar_temp = gevent.event.AsyncResult()
def subscriber2(m, r, s):
ar_temp.set(m)
sub_temp = StandaloneStreamSubscriber('sub_temp', subscriber2)
self.addCleanup(sub_temp.stop)
ar_pres = gevent.event.AsyncResult()
def subscriber3(m, r, s):
ar_pres.set(m)
sub_pres = StandaloneStreamSubscriber('sub_pres', subscriber3)
self.addCleanup(sub_pres.stop)
sub_cond_id = self.pubsub.create_subscription(
'subscription_cond',
stream_ids=[cond_stream_id],
exchange_name='sub_cond')
sub_temp_id = self.pubsub.create_subscription(
'subscription_temp',
stream_ids=[temp_stream_id],
exchange_name='sub_temp')
sub_pres_id = self.pubsub.create_subscription(
'subscription_pres',
stream_ids=[pres_stream_id],
exchange_name='sub_pres')
self.pubsub.activate_subscription(sub_cond_id)
self.pubsub.activate_subscription(sub_temp_id)
self.pubsub.activate_subscription(sub_pres_id)
self.queue_cleanup.append(sub_cond.xn.queue)
self.queue_cleanup.append(sub_temp.xn.queue)
self.queue_cleanup.append(sub_pres.xn.queue)
sub_cond.start()
sub_temp.start()
sub_pres.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
publish_granule = self._get_new_ctd_packet(
stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result_cond = ar_cond.get(timeout=10)
result_temp = ar_temp.get(timeout=10)
result_pres = ar_pres.get(timeout=10)
out_dict = {}
out_dict['c'] = RecordDictionaryTool.load_from_granule(
result_cond)['conductivity']
out_dict['t'] = RecordDictionaryTool.load_from_granule(
result_temp)['temp']
out_dict['p'] = RecordDictionaryTool.load_from_granule(
result_pres)['pressure']
# Check that the transform algorithm was successfully executed
self.check_granule_splitting(publish_granule, out_dict)
def test_demux(self):
self.stream0, self.route0 = self.pubsub_client.create_stream(
'stream0', exchange_point='test')
self.stream1, self.route1 = self.pubsub_client.create_stream(
'stream1', exchange_point='main_data')
self.stream2, self.route2 = self.pubsub_client.create_stream(
'stream2', exchange_point='alt_data')
self.r_stream1 = gevent.event.Event()
self.r_stream2 = gevent.event.Event()
def process(msg, stream_route, stream_id):
if stream_id == self.stream1:
self.r_stream1.set()
elif stream_id == self.stream2:
self.r_stream2.set()
self.container.spawn_process(
'demuxer', 'ion.processes.data.transforms.mux', 'DemuxTransform',
{'process': {
'out_streams': [self.stream1, self.stream2]
}}, 'demuxer_pid')
self.queue_cleanup.append('demuxer_pid')
sub1 = StandaloneStreamSubscriber('sub1', process)
sub2 = StandaloneStreamSubscriber('sub2', process)
sub1.xn.bind(self.route1.routing_key,
self.container.ex_manager.create_xp('main_data'))
sub2.xn.bind(self.route2.routing_key,
self.container.ex_manager.create_xp('alt_data'))
sub1.start()
sub2.start()
self.queue_cleanup.append(sub1.xn)
self.queue_cleanup.append(sub2.xn)
xn = self.container.ex_manager.create_xn_queue('demuxer_pid')
xn.bind(
self.route0.routing_key,
self.container.ex_manager.create_xp(self.route0.exchange_point))
domino = StandaloneStreamPublisher(self.stream0, self.route0)
domino.publish('test')
self.assertTrue(self.r_stream1.wait(2))
self.assertTrue(self.r_stream2.wait(2))
self.container.proc_manager.terminate_process('demuxer_pid')
sub1.stop()
sub2.stop()
def test_full_pubsub(self):
self.sub1_sat = Event()
self.sub2_sat = Event()
def subscriber1(m, r, s):
self.sub1_sat.set()
def subscriber2(m, r, s):
self.sub2_sat.set()
sub1 = StandaloneStreamSubscriber('sub1', subscriber1)
self.queue_cleanup.append(sub1.xn.queue)
sub1.start()
sub2 = StandaloneStreamSubscriber('sub2', subscriber2)
self.queue_cleanup.append(sub2.xn.queue)
sub2.start()
log_topic = self.pubsub_management.create_topic(
'instrument_logs', exchange_point='instruments')
science_topic = self.pubsub_management.create_topic(
'science_data', exchange_point='instruments')
events_topic = self.pubsub_management.create_topic(
'notifications', exchange_point='events')
log_stream, route = self.pubsub_management.create_stream(
'instrument1-logs',
topic_ids=[log_topic],
exchange_point='instruments')
ctd_stream, route = self.pubsub_management.create_stream(
'instrument1-ctd',
topic_ids=[science_topic],
exchange_point='instruments')
event_stream, route = self.pubsub_management.create_stream(
'notifications', topic_ids=[events_topic], exchange_point='events')
raw_stream, route = self.pubsub_management.create_stream(
'temp', exchange_point='global.data')
self.exchange_cleanup.extend(['instruments', 'events', 'global.data'])
subscription1 = self.pubsub_management.create_subscription(
'subscription1',
stream_ids=[log_stream, event_stream],
exchange_name='sub1')
subscription2 = self.pubsub_management.create_subscription(
'subscription2',
exchange_points=['global.data'],
stream_ids=[ctd_stream],
exchange_name='sub2')
self.pubsub_management.activate_subscription(subscription1)
self.pubsub_management.activate_subscription(subscription2)
self.publish_on_stream(log_stream, 1)
self.assertTrue(self.sub1_sat.wait(4))
self.assertFalse(self.sub2_sat.is_set())
self.publish_on_stream(raw_stream, 1)
self.assertTrue(self.sub1_sat.wait(4))
sub1.stop()
sub2.stop()
def setup_validator(self, validator):
listener = StandaloneStreamSubscriber('listener', validator)
listener.start()
self.addCleanup(listener.stop)
def test_replay_pause(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
scov = DatasetManagementService._get_simplex_coverage(dataset_id)
bb = CoverageCraft(scov)
bb.rdt['time'] = np.arange(100)
bb.rdt['temp'] = np.random.random(100) + 30
bb.sync_with_granule()
DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations
# Set up the subscriber to verify the data
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
xp = self.container.ex_manager.create_xp('xp1')
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
# Set up the replay agent and the client wrapper
# 1) Define the Replay (dataset and stream to publish on)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream)
# 2) Make a client to the interact with the process (optionall provide it a process to bind with)
replay_client = ReplayClient(process_id)
# 3) Start the agent (launch the process)
self.data_retriever.start_replay_agent(self.replay_id)
# 4) Start replaying...
replay_client.start_replay()
# Wait till we get some granules
self.assertTrue(self.event.wait(5))
# We got granules, pause the replay, clear the queue and allow the process to finish consuming
replay_client.pause_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure there's no remaining messages being consumed
self.assertFalse(self.event.wait(1))
# Resume the replay and wait until we start getting granules again
replay_client.resume_replay()
self.assertTrue(self.event.wait(5))
# Stop the replay, clear the queues
replay_client.stop_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure that it did indeed stop
self.assertFalse(self.event.wait(1))
subscriber.stop()
def test_granule(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition(
'ctd', parameter_dictionary_id=pdict_id)
pdict = DatasetManagementService.get_parameter_dictionary_by_name(
'ctd_parsed_param_dict')
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
stream_id, route = self.pubsub_management.create_stream(
'ctd_stream', 'xp1', stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
self.xps.append('xp1')
publisher = StandaloneStreamPublisher(stream_id, route)
subscriber = StandaloneStreamSubscriber('sub', self.verify_incoming)
subscriber.start()
subscription_id = self.pubsub_management.create_subscription(
'sub', stream_ids=[stream_id])
self.xns.append('sub')
self.pubsub_management.activate_subscription(subscription_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.random.randn(10) * 10 + 30
rdt['pressure'] = [20] * 10
self.assertEquals(set(pdict.keys()), set(rdt.fields))
self.assertEquals(pdict.temporal_parameter_name,
rdt.temporal_parameter)
self.rdt = rdt
self.data_producer_id = 'data_producer'
self.provider_metadata_update = {1: 1}
publisher.publish(
rdt.to_granule(data_producer_id='data_producer',
provider_metadata_update={1: 1}))
self.assertTrue(self.event.wait(10))
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
filtered_stream_def_id = self.pubsub_management.create_stream_definition(
'filtered',
parameter_dictionary_id=pdict_id,
available_fields=['time', 'temp'])
self.addCleanup(self.pubsub_management.delete_stream_definition,
filtered_stream_def_id)
rdt = RecordDictionaryTool(stream_definition_id=filtered_stream_def_id)
self.assertEquals(rdt._available_fields, ['time', 'temp'])
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
with self.assertRaises(KeyError):
rdt['pressure'] = np.arange(20)
granule = rdt.to_granule()
rdt2 = RecordDictionaryTool.load_from_granule(granule)
self.assertEquals(rdt._available_fields, rdt2._available_fields)
self.assertEquals(rdt.fields, rdt2.fields)
for k, v in rdt.iteritems():
self.assertTrue(np.array_equal(rdt[k], rdt2[k]))
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.array([None, None, None])
self.assertTrue(rdt['time'] is None)
rdt['time'] = np.array([None, 1, 2])
self.assertEquals(rdt['time'][0], rdt.fill_value('time'))
def test_topic_craziness(self):
self.msg_queue = Queue()
def subscriber1(m, r, s):
self.msg_queue.put(m)
sub1 = StandaloneStreamSubscriber('sub1', subscriber1)
self.queue_cleanup.append(sub1.xn.queue)
sub1.start()
topic1 = self.pubsub_management.create_topic('topic1',
exchange_point='xp1')
topic2 = self.pubsub_management.create_topic('topic2',
exchange_point='xp1',
parent_topic_id=topic1)
topic3 = self.pubsub_management.create_topic('topic3',
exchange_point='xp1',
parent_topic_id=topic1)
topic4 = self.pubsub_management.create_topic('topic4',
exchange_point='xp1',
parent_topic_id=topic2)
topic5 = self.pubsub_management.create_topic('topic5',
exchange_point='xp1',
parent_topic_id=topic2)
topic6 = self.pubsub_management.create_topic('topic6',
exchange_point='xp1',
parent_topic_id=topic3)
topic7 = self.pubsub_management.create_topic('topic7',
exchange_point='xp1',
parent_topic_id=topic3)
# Tree 2
topic8 = self.pubsub_management.create_topic('topic8',
exchange_point='xp2')
topic9 = self.pubsub_management.create_topic('topic9',
exchange_point='xp2',
parent_topic_id=topic8)
topic10 = self.pubsub_management.create_topic('topic10',
exchange_point='xp2',
parent_topic_id=topic9)
topic11 = self.pubsub_management.create_topic('topic11',
exchange_point='xp2',
parent_topic_id=topic9)
topic12 = self.pubsub_management.create_topic('topic12',
exchange_point='xp2',
parent_topic_id=topic11)
topic13 = self.pubsub_management.create_topic('topic13',
exchange_point='xp2',
parent_topic_id=topic11)
self.exchange_cleanup.extend(['xp1', 'xp2'])
stream1_id, route = self.pubsub_management.create_stream(
'stream1',
topic_ids=[topic7, topic4, topic5],
exchange_point='xp1')
stream2_id, route = self.pubsub_management.create_stream(
'stream2', topic_ids=[topic8], exchange_point='xp2')
stream3_id, route = self.pubsub_management.create_stream(
'stream3', topic_ids=[topic10, topic13], exchange_point='xp2')
stream4_id, route = self.pubsub_management.create_stream(
'stream4', topic_ids=[topic9], exchange_point='xp2')
stream5_id, route = self.pubsub_management.create_stream(
'stream5', topic_ids=[topic11], exchange_point='xp2')
subscription1 = self.pubsub_management.create_subscription(
'sub1', topic_ids=[topic1])
subscription2 = self.pubsub_management.create_subscription(
'sub2', topic_ids=[topic8], exchange_name='sub1')
subscription3 = self.pubsub_management.create_subscription(
'sub3', topic_ids=[topic9], exchange_name='sub1')
subscription4 = self.pubsub_management.create_subscription(
'sub4',
topic_ids=[topic10, topic13, topic11],
exchange_name='sub1')
#--------------------------------------------------------------------------------
self.pubsub_management.activate_subscription(subscription1)
self.publish_on_stream(stream1_id, 1)
self.assertEquals(self.msg_queue.get(timeout=10), 1)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.1)
self.pubsub_management.deactivate_subscription(subscription1)
self.pubsub_management.delete_subscription(subscription1)
#--------------------------------------------------------------------------------
self.pubsub_management.activate_subscription(subscription2)
self.publish_on_stream(stream2_id, 2)
self.assertEquals(self.msg_queue.get(timeout=10), 2)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.1)
self.pubsub_management.deactivate_subscription(subscription2)
self.pubsub_management.delete_subscription(subscription2)
#--------------------------------------------------------------------------------
self.pubsub_management.activate_subscription(subscription3)
self.publish_on_stream(stream2_id, 3)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.3)
self.publish_on_stream(stream3_id, 4)
self.assertEquals(self.msg_queue.get(timeout=10), 4)
self.pubsub_management.deactivate_subscription(subscription3)
self.pubsub_management.delete_subscription(subscription3)
#--------------------------------------------------------------------------------
self.pubsub_management.activate_subscription(subscription4)
self.publish_on_stream(stream4_id, 5)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.3)
self.publish_on_stream(stream5_id, 6)
self.assertEquals(self.msg_queue.get(timeout=10), 6)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.3)
self.pubsub_management.deactivate_subscription(subscription4)
self.pubsub_management.delete_subscription(subscription4)
#--------------------------------------------------------------------------------
sub1.stop()
self.pubsub_management.delete_topic(topic13)
self.pubsub_management.delete_topic(topic12)
self.pubsub_management.delete_topic(topic11)
self.pubsub_management.delete_topic(topic10)
self.pubsub_management.delete_topic(topic9)
self.pubsub_management.delete_topic(topic8)
self.pubsub_management.delete_topic(topic7)
self.pubsub_management.delete_topic(topic6)
self.pubsub_management.delete_topic(topic5)
self.pubsub_management.delete_topic(topic4)
self.pubsub_management.delete_topic(topic3)
self.pubsub_management.delete_topic(topic2)
self.pubsub_management.delete_topic(topic1)
self.pubsub_management.delete_stream(stream1_id)
self.pubsub_management.delete_stream(stream2_id)
self.pubsub_management.delete_stream(stream3_id)
self.pubsub_management.delete_stream(stream4_id)
self.pubsub_management.delete_stream(stream5_id)
def test_ctdbp_L0_all(self):
"""
Test packets processed by the ctdbp_L0_all transform
"""
#----------- Data Process Definition --------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='CTDBP_L0_all',
description='Take parsed stream and put the C, T and P into three separate L0 streams.',
module='ion.processes.data.transforms.ctdbp.ctdbp_L0',
class_name='CTDBP_L0_all')
dprocdef_id = self.data_process_management.create_data_process_definition(dpd_obj)
self.addCleanup(self.data_process_management.delete_data_process_definition, dprocdef_id)
log.debug("created data process definition: id = %s", dprocdef_id)
#----------- Data Products --------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_param_dict = self._create_input_param_dict_for_test(parameter_dict_name = 'fictitious_ctdp_param_dict')
# Get the stream definition for the stream using the parameter dictionary
# input_param_dict = self.dataset_management.read_parameter_dictionary_by_name('ctdbp_cdef_sample', id_only=True)
input_stream_def_dict = self.pubsub.create_stream_definition(name='parsed', parameter_dictionary_id=input_param_dict)
self.addCleanup(self.pubsub.delete_stream_definition, input_stream_def_dict)
log.debug("Got the parsed parameter dictionary: id: %s", input_param_dict)
log.debug("Got the stream def for parsed input: %s", input_stream_def_dict)
# Input data product
parsed_stream_dp_obj = IonObject(RT.DataProduct,
name='parsed_stream',
description='Parsed stream input to CTBP L0 transform',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproduct_management.create_data_product(data_product=parsed_stream_dp_obj,
stream_definition_id=input_stream_def_dict
)
self.addCleanup(self.dataproduct_management.delete_data_product, input_dp_id)
# output data product
L0_stream_dp_obj = IonObject(RT.DataProduct,
name='L0_stream',
description='L0_stream output of CTBP L0 transform',
temporal_domain = tdom,
spatial_domain = sdom)
L0_stream_dp_id = self.dataproduct_management.create_data_product(data_product=L0_stream_dp_obj,
stream_definition_id=input_stream_def_dict
)
self.addCleanup(self.dataproduct_management.delete_data_product, L0_stream_dp_id)
# We need the key name here to be "L0_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L0_stream when the config is used to launch the data process
self.output_products = {'L0_stream' : L0_stream_dp_id}
out_stream_ids, _ = self.resource_registry.find_objects(L0_stream_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(out_stream_ids))
output_stream_id = out_stream_ids[0]
dproc_id = self.data_process_management.create_data_process( dprocdef_id, [input_dp_id], self.output_products)
self.addCleanup(self.data_process_management.delete_data_process, dproc_id)
log.debug("Created a data process for ctdbp_L0. id: %s", dproc_id)
# Activate the data process
self.data_process_management.activate_data_process(dproc_id)
self.addCleanup(self.data_process_management.deactivate_data_process, dproc_id)
#----------- Find the stream that is associated with the input data product when it was created by create_data_product() --------------------------------
stream_ids, _ = self.resource_registry.find_objects(input_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(stream_ids))
input_stream_id = stream_ids[0]
stream_route = self.pubsub.read_stream_route(input_stream_id)
log.debug("The input stream for the L0 transform: %s", input_stream_id)
#----------- Create a subscriber that will listen to the transform's output --------------------------------
ar = gevent.event.AsyncResult()
def subscriber(m,r,s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub', callback=subscriber)
sub_id = self.pubsub.create_subscription('subscriber_to_transform',
stream_ids=[output_stream_id],
exchange_name='sub')
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
#----------- Publish on that stream so that the transform can receive it --------------------------------
pub = StandaloneStreamPublisher(input_stream_id, stream_route)
publish_granule = self._get_new_ctd_packet(stream_definition_id=input_stream_def_dict, length = 5)
pub.publish(publish_granule)
log.debug("Published the following granule: %s", publish_granule)
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the transform: %s", granule_from_transform)
# Check that the granule published by the L0 transform has the right properties
self._check_granule_from_transform(granule_from_transform)
def test_ctd_L2_salinity(self):
'''
Test that packets are processed by the ctd_L1_salinity transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='SalinityTransform',
description='For testing SalinityTransform')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.ctd_L2_salinity'
process_definition.executable['class'] = 'SalinityTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('sal_stream_def', parameter_dictionary_id=pdict_id)
sal_stream_id, _ = self.pubsub.create_stream('test_salinity', stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.salinity = sal_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
#---------------------------------------------------------------------------------------------
# Create a subscriber that will receive the salinity granule from the ctd transform
#---------------------------------------------------------------------------------------------
ar_sal = gevent.event.AsyncResult()
def subscriber3(m,r,s):
ar_sal.set(m)
sub_sal = StandaloneStreamSubscriber('sub_sal', subscriber3)
self.addCleanup(sub_sal.stop)
sub_sal_id = self.pubsub.create_subscription('subscription_sal',
stream_ids=[sal_stream_id],
exchange_name='sub_sal')
self.pubsub.activate_subscription(sub_sal_id)
self.queue_cleanup.append(sub_sal.xn.queue)
sub_sal.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_sal.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('salinity'))
self.check_salinity_algorithm_execution(publish_granule, result)
def instrument_test_driver(container):
sa_user_header = container.governance_controller.get_system_actor_header()
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'mi.instrument.seabird.sbe37smb.ooicore.driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
_container_client = ContainerAgentClient(node=container.node,
name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, *args, **kwargs):
log.info('Subscriber received message: %s', str(message))
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
#@TODO: Figure out what goes inside this stream def, do we have a pdict?
stream_def_id = _pubsub_client.create_stream_definition(
name='instrument stream def')
stream_id, route = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
exchange_point='science_data')
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = StandaloneStreamSubscriber(exchange_name=exchange_name, callback=consume)
sub.start()
sub_id = _pubsub_client.create_subscription(\
name=exchange_name,
stream_ids=[stream_id])
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = '123xyz'
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': agent_resource_id}
}
# Launch an instrument agent process.
_ia_name = 'agent007'
_ia_mod = 'ion.agents.instrument.instrument_agent'
_ia_class = 'InstrumentAgent'
_ia_pid = _container_client.spawn_process(name=_ia_name,
module=_ia_mod, cls=_ia_class,
config=agent_config)
log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
def test_ctd_L2_salinity(self):
'''
Test that packets are processed by the ctd_L1_salinity transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='SalinityTransform',
description='For testing SalinityTransform')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.ctd_L2_salinity'
process_definition.executable['class'] = 'SalinityTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'sal_stream_def', parameter_dictionary_id=pdict_id)
sal_stream_id, _ = self.pubsub.create_stream(
'test_salinity',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.salinity = sal_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Create a subscriber that will receive the salinity granule from the ctd transform
#---------------------------------------------------------------------------------------------
ar_sal = gevent.event.AsyncResult()
def subscriber3(m, r, s):
ar_sal.set(m)
sub_sal = StandaloneStreamSubscriber('sub_sal', subscriber3)
self.addCleanup(sub_sal.stop)
sub_sal_id = self.pubsub.create_subscription(
'subscription_sal',
stream_ids=[sal_stream_id],
exchange_name='sub_sal')
self.pubsub.activate_subscription(sub_sal_id)
self.queue_cleanup.append(sub_sal.xn.queue)
sub_sal.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
publish_granule = self._get_new_ctd_packet(
stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_sal.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('salinity'))
self.check_salinity_algorithm_execution(publish_granule, result)
def test_event_triggered_transform_A(self):
'''
Test that packets are processed by the event triggered transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='EventTriggeredTransform_A',
description='For testing EventTriggeredTransform_A')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.event_triggered_transform'
process_definition.executable['class'] = 'EventTriggeredTransform_A'
event_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'cond_stream_def', parameter_dictionary_id=pdict_id)
cond_stream_id, _ = self.pubsub.create_stream(
'test_conductivity',
exchange_point='science_data',
stream_definition_id=stream_def_id)
config.process.publish_streams.conductivity = cond_stream_id
config.process.event_type = 'ResourceLifecycleEvent'
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=event_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Publish an event to wake up the event triggered transform
#---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ResourceLifecycleEvent")
event_publisher.publish_event(origin='fake_origin')
#---------------------------------------------------------------------------------------------
# Create subscribers that will receive the conductivity, temperature and pressure granules from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber1(m, r, s):
ar_cond.set(m)
sub_event_transform = StandaloneStreamSubscriber(
'sub_event_transform', subscriber1)
self.addCleanup(sub_event_transform.stop)
sub_event_transform_id = self.pubsub.create_subscription(
'subscription_cond',
stream_ids=[cond_stream_id],
exchange_name='sub_event_transform')
self.pubsub.activate_subscription(sub_event_transform_id)
self.queue_cleanup.append(sub_event_transform.xn.queue)
sub_event_transform.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(
stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result_cond = ar_cond.get(timeout=10)
self.assertTrue(isinstance(result_cond, Granule))
rdt = RecordDictionaryTool.load_from_granule(result_cond)
self.assertTrue(rdt.__contains__('conductivity'))
self.check_cond_algorithm_execution(publish_granule, result_cond)
