def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info("Subscriber received data message: %s.", str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container, node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name, stream_definition_id=stream_def_id, original=True, encoding="ION R2"
)
self._stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = "%s_queue" % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(query=query, exchange_name=exchange_name)
pubsub_client.activate_subscription(sub_id)
def on_start(self):
pubsub_cli = PubsubManagementServiceProcessClient(process=self, node=self.container.node)
# Get the stream(s)
stream_id = self.CFG.get_safe('process.stream_id','')
query = StreamQuery(stream_ids=[stream_id,])
exchange_name = 'dispatcher_%s' % self.id
subscription_id = pubsub_cli.create_subscription(
query = query,
exchange_name = exchange_name,
name = "SampleSubscription",
description = "Sample Subscription Description")
stream_subscriber = StreamSubscriberRegistrar(process=self, node=self.container.node)
def message_received(granule, h):
rdt = RecordDictionaryTool.load_from_granule(granule)
log.warn('Logging Record Dictionary received in logger subscription \n%s', rdt.pretty_print())
subscriber = stream_subscriber.create_subscriber(exchange_name=exchange_name, callback=message_received)
subscriber.start()
pubsub_cli.activate_subscription(subscription_id)
def start_output_stream_and_listen(self, ctd_stream_id, data_product_stream_ids, message_count_per_stream=10):
cc = self.container
assertions = self.assertTrue
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name = 'workflow_test',
exchange_point = 'science_data',
name = "test workflow transformations",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, container=cc)
result = gevent.event.AsyncResult()
results = []
message_count = len(data_product_stream_ids) * message_count_per_stream
def message_received(message, headers):
# Heads
results.append(message)
if len(results) >= message_count: #Only wait for so many messages - per stream
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='workflow_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
#Start the input stream process
if ctd_stream_id is not None:
ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id)
# Assert that we have received data
assertions(result.get(timeout=30))
# stop the flow parse the messages...
if ctd_stream_id is not None:
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def _start_output_stream_listener(self,
data_product_stream_ids,
message_count_per_stream=10):
cc = self.container
assertions = self.assertTrue
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name='workflow_test',
name="test workflow transformations",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process,
node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn(' data received!')
results.append(message)
if len(results) >= len(
data_product_stream_ids
) * message_count_per_stream: #Only wait for so many messages - per stream
result.set(True)
subscriber = subscriber_registrar.create_subscriber(
exchange_name='workflow_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(
subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=30))
self.pubsubclient.deactivate_subscription(
subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
container=self.container)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
# TODO the following is a mininal adjustment to at least let the test
# continue:
# for (stream_name, val) in PACKET_CONFIG.iteritems():
for stream_name in PACKET_CONFIG:
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(
container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
taxy = get_taxonomy(stream_name)
stream_config = dict(
id=stream_id,
taxonomy=taxy.dump()
)
self._stream_config[stream_name] = stream_config
# self._stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(
query=query, exchange_name=exchange_name, exchange_point='science_data')
pubsub_client.activate_subscription(sub_id)
def test_chop_chop(self):
# Override couch
self.couch = self.container.datastore_manager.get_datastore(
ds_name='chopping_block',
profile=DataStore.DS_PROFILE.SCIDATA
)
self.datastore_name = 'chopping_block'
granule = ctd_stream_packet(
stream_id='this_is_only_a_test',
time='12345', #Same combo on my luggage
create_hdf=False
)
self.couch.create(granule)
log.debug("Granule: %s", granule)
dataset_id = self.dsm_cli.create_dataset(
stream_id='this_is_only_a_test',
datastore_name=self.datastore_name,
view_name='datasets/dataset_by_id',
name='sci_data_granule_chop'
)
replay_id, stream_id = self.dr_cli.define_replay(
dataset_id=dataset_id,
delivery_format={'chop':True}
)
replay = self.rr_cli.read(replay_id)
self.assertTrue(self.container.proc_manager.procs[replay.process_id])
async_result = gevent.event.AsyncResult()
def consume(message, headers):
async_result.set(message)
stream_subscriber = StreamSubscriberRegistrar(process=self.container, node=self.container.node)
subscriber = stream_subscriber.create_subscriber(exchange_name = 'chopping_block', callback=consume)
subscriber.start()
query = StreamQuery(stream_ids=[stream_id])
subscription_id = self.ps_cli.create_subscription(query=query, exchange_name='chopping_block')
self.ps_cli.activate_subscription(subscription_id=subscription_id)
self.dr_cli.start_replay(replay_id)
for fields in xrange(4):
self.assertTrue(async_result.get(timeout=10))
subscriber.stop()
self.dr_cli.cancel_replay(replay_id=replay_id)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1",
description="Sample Stream 1 Description")
self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2",
description="Sample Stream 2 Description")
# Make a subscription to two input streams
exchange_name = "a_queue"
query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id])
self.ctd_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleSubscription",
"Sample Subscription Description")
# Make a subscription to all streams on an exchange point
exchange_name = "another_queue"
query = ExchangeQuery()
self.exchange_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleExchangeSubscription",
"Sample Exchange Subscription Description")
pid = self.container.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = self.container.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.container.node)
self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream1_id)
self.ctd_stream2_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream2_id)
# Cheat and use the cc as the process - I don't think it is used for anything...
self.stream_subscriber = StreamSubscriberRegistrar(process=dummy_process, node=self.container.node)
def __init__(self, packet_config = {}, stream_definition = None, original = True, encoding = "ION R2"):
log.info("Start data subscribers")
self.no_samples = None
self.async_data_result = AsyncResult()
self.data_greenlets = []
self.stream_config = {}
self.samples_received = []
self.data_subscribers = []
self.container = Container.instance
if not self.container:
raise NoContainer()
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self.samples_received.append(message)
if self.no_samples and self.no_samples == len(self.samples_received):
self.async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container, node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
self.data_subscribers = []
for (stream_name, val) in packet_config.iteritems():
stream_def_id = pubsub_client.create_stream_definition(container=stream_definition)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=original,
encoding=encoding)
self.stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume_data)
self._listen(sub)
self.data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(query=query, exchange_name=exchange_name)
pubsub_client.activate_subscription(sub_id)
def _start_output_stream_listener(self, data_product_stream_ids, message_count_per_stream=10):
cc = self.container
assertions = self.assertTrue
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name = 'workflow_test',
name = "test workflow transformations",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn(' data received!')
results.append(message)
if len(results) >= len(data_product_stream_ids) * message_count_per_stream: #Only wait for so many messages - per stream
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='workflow_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=30))
self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(
container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self._stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name)
pubsub_client.activate_subscription(sub_id)
def setUp(self):
self._start_container()
self.cc = ContainerAgentClient(node=self.container.node,name=self.container.name)
self.cc.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.cc.node)
self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1",
description="Sample Stream 1 Description")
self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2",
description="Sample Stream 2 Description")
# Make a subscription to two input streams
exchange_name = "a_queue"
query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id])
self.ctd_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleSubscription",
"Sample Subscription Description")
# Make a subscription to all streams on an exchange point
exchange_name = "another_queue"
query = ExchangeQuery()
self.exchange_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleExchangeSubscription",
"Sample Exchange Subscription Description")
pid = self.container.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = self.container.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.cc.node)
self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream1_id)
self.ctd_stream2_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream2_id)
# Cheat and use the cc as the process - I don't think it is used for anything...
self.stream_subscriber = StreamSubscriberRegistrar(process=dummy_process, node=self.cc.node)
def test_start_replay(self):
post = BlogPost(title='test blog post', post_id='12345', author=BlogAuthor(name='Jon Doe'), content='this is a blog post',
updated=time.strftime("%Y-%m-%dT%H:%M%S-05"))
dataset_id = self.dsm_cli.create_dataset(
stream_id='12345',
datastore_name=self.datastore_name,
view_name='posts/posts_join_comments',
name='blog posts test'
)
self.couch.create(post)
replay_id, stream_id = self.dr_cli.define_replay(dataset_id)
replay = self.rr_cli.read(replay_id)
# assert that the process was created
self.assertTrue(self.container.proc_manager.procs[replay.process_id])
# pattern from Tim G
ar = gevent.event.AsyncResult()
def consume(message, headers):
ar.set(message)
stream_subscriber = StreamSubscriberRegistrar(process=self.container, node=self.container.node)
subscriber = stream_subscriber.create_subscriber(exchange_name='test_queue', callback=consume)
subscriber.start()
query = StreamQuery(stream_ids=[stream_id])
subscription_id = self.ps_cli.create_subscription(query=query,exchange_name='test_queue')
self.ps_cli.activate_subscription(subscription_id)
self.dr_cli.start_replay(replay_id)
self.assertEqual(ar.get(timeout=10).post_id,post.post_id)
subscriber.stop()
def on_start(self):
pubsub_cli = PubsubManagementServiceProcessClient(
process=self, node=self.container.node)
# Get the stream(s)
stream_id = self.CFG.get_safe('process.stream_id', '')
query = StreamQuery(stream_ids=[
stream_id,
])
exchange_name = 'dispatcher_%s' % self.id
subscription_id = pubsub_cli.create_subscription(
query=query,
exchange_name=exchange_name,
name="SampleSubscription",
description="Sample Subscription Description")
stream_subscriber = StreamSubscriberRegistrar(process=self,
node=self.container.node)
def message_received(granule, h):
rdt = RecordDictionaryTool.load_from_granule(granule)
log.warn(
'Logging Record Dictionary received in logger subscription \n%s',
rdt.pretty_print())
subscriber = stream_subscriber.create_subscriber(
exchange_name=exchange_name, callback=message_received)
subscriber.start()
pubsub_cli.activate_subscription(subscription_id)
def on_start(self):
rr_cli = ResourceRegistryServiceProcessClient(process=self, node=self.container.node)
pubsub_cli = PubsubManagementServiceProcessClient(process=self, node=self.container.node)
# Get the stream(s)
data_product_id = self.CFG.get_safe('dispatcher.data_product_id','')
stream_ids,_ = rr_cli.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
log.info('Got Stream Ids: "%s"', stream_ids)
assert stream_ids, 'No streams found for this data product!'
query = StreamQuery(stream_ids=stream_ids)
exchange_name = 'dispatcher_%s' % str(os.getpid())
subscription_id = pubsub_cli.create_subscription(
query = query,
exchange_name = exchange_name,
name = "SampleSubscription",
description = "Sample Subscription Description")
stream_subscriber = StreamSubscriberRegistrar(process=self, node=self.container.node)
stream_defs = {}
def message_received(granule, h):
stream_id = granule.stream_resource_id
data_stream_id = granule.data_stream_id
data_stream = granule.identifiables[data_stream_id]
tstamp = get_datetime(data_stream.timestamp.value)
records = granule.identifiables['record_count'].value
log.info('Received a message from stream %s with time stamp %s and %d records' % (stream_id, tstamp, records))
if stream_id not in stream_defs:
stream_defs[stream_id] = pubsub_cli.find_stream_definition(stream_id, id_only=False).container
stream_def = stream_defs.get(stream_id)
sp = PointSupplementStreamParser(stream_definition=stream_def, stream_granule=granule)
last_data = {}
for field in sp.list_field_names():
last_data[field] = sp.get_values(field)[-1]
log.info('Last values in the message: %s' % str(last_data))
subscriber = stream_subscriber.create_subscriber(exchange_name=exchange_name, callback=message_received)
subscriber.start()
pubsub_cli.activate_subscription(subscription_id)
def test_replay_integration(self):
'''
Test full DM Services Integration
'''
cc = self.container
### Every thing below here can be run as a script:
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(node=cc.node)
dataset_management_service = DatasetManagementServiceClient(node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
resource_registry_service = ResourceRegistryServiceClient(node=cc.node)
#------------------------------------------------------------------------------------------------------
# Datastore name
#------------------------------------------------------------------------------------------------------
datastore_name = 'test_replay_integration'
#------------------------------------------------------------------------------------------------------
# Spawn process
#------------------------------------------------------------------------------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
#------------------------------------------------------------------------------------------------------
# Set up subscriber
#------------------------------------------------------------------------------------------------------
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
#------------------------------------------------------------------------------------------------------
# Set up ingestion
#------------------------------------------------------------------------------------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name, datastore_profile='SCIDATA'),
hdf_storage=HdfStorage(),
number_of_workers=1,
)
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
#------------------------------------------------------------------------------------------------------
# Grab the transforms acting as ingestion workers
#------------------------------------------------------------------------------------------------------
transforms = [resource_registry_service.read(assoc.o)
for assoc in resource_registry_service.find_associations(ingestion_configuration_id, PRED.hasTransform)]
proc_1 = cc.proc_manager.procs[transforms[0].process_id]
log.info("PROCESS 1: %s" % str(proc_1))
#------------------------------------------------------------------------------------------------------
# Set up the test hooks for the gevent event AsyncResult object
#------------------------------------------------------------------------------------------------------
def ingestion_worker_received(message, headers):
ar.set(message)
proc_1.ingest_process_test_hook = ingestion_worker_received
#------------------------------------------------------------------------------------------------------
# Set up the producers (CTD Simulators)
#------------------------------------------------------------------------------------------------------
ctd_stream_def = ctd_stream_definition()
stream_def_id = pubsub_management_service.create_stream_definition(container=ctd_stream_def, name='Junk definition')
stream_id = pubsub_management_service.create_stream(stream_definition_id=stream_def_id)
#------------------------------------------------------------------------------------------------------
# Set up the dataset config
#------------------------------------------------------------------------------------------------------
dataset_id = dataset_management_service.create_dataset(
stream_id=stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule'
)
dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id = dataset_id,
archive_data = True,
archive_metadata = True,
ingestion_configuration_id = ingestion_configuration_id
)
#------------------------------------------------------------------------------------------------------
# Launch a ctd_publisher
#------------------------------------------------------------------------------------------------------
publisher = publisher_registrar.create_publisher(stream_id=stream_id)
#------------------------------------------------------------------------
# Create a packet and publish it
#------------------------------------------------------------------------
ctd_packet = _create_packet(stream_id)
published_hdfstring = ctd_packet.identifiables['ctd_data'].values
publisher.publish(ctd_packet)
#------------------------------------------------------------------------------------------------------
# Catch what the ingestion worker gets! Assert it is the same packet that was published!
#------------------------------------------------------------------------------------------------------
packet = ar.get(timeout=2)
#------------------------------------------------------------------------------------------------------
# Create subscriber to listen to the replays
#------------------------------------------------------------------------------------------------------
replay_id, replay_stream_id = data_retriever_service.define_replay(dataset_id)
query = StreamQuery(stream_ids=[replay_stream_id])
subscription_id = pubsub_management_service.create_subscription(query = query, exchange_name='replay_capture_point' ,name = 'replay_capture_point')
# It is not required or even generally a good idea to use the subscription resource name as the queue name, but it makes things simple here
# Normally the container creates and starts subscribers for you when a transform process is spawned
subscriber = subscriber_registrar.create_subscriber(exchange_name='replay_capture_point', callback=_subscriber_call_back)
subscriber.start()
pubsub_management_service.activate_subscription(subscription_id)
#------------------------------------------------------------------------------------------------------
# Start the replay
#------------------------------------------------------------------------------------------------------
data_retriever_service.start_replay(replay_id)
#------------------------------------------------------------------------------------------------------
# Get the hdf string from the captured stream in the replay
#------------------------------------------------------------------------------------------------------
retrieved_hdf_string = ar2.get(timeout=2)
### Non scriptable portion of the test
#------------------------------------------------------------------------------------------------------
# Assert that it matches the message we sent
#------------------------------------------------------------------------------------------------------
self.assertEquals(packet.identifiables['stream_encoding'].sha1, ctd_packet.identifiables['stream_encoding'].sha1)
self.assertEquals(retrieved_hdf_string, published_hdfstring)
def test_workflow_components(self):
cc = self.container
assertions = self.assertTrue
#-------------------------------
# Create CTD Parsed as the initial data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data')
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except Exception as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
ctd_stream_id = stream_ids[0]
###
### Setup the first transformation
###
# Salinity: Data Process Definition
log.debug("Create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_salinity',
description='create a salinity data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Excpetion as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(sal_stream_def_id, ctd_L2_salinity_dprocdef_id )
# Create the output data product of the transform
log.debug("create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity',description='transform output L2 salinity')
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, sal_stream_def_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id, persist_data=True, persist_metadata=True)
# Create the Salinity transform data process
log.debug("create L2_salinity data_process and start it")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product, {'output':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
###
### Setup the second transformation
###
# Salinity: Data Process Definition
log.debug("Create data process definition SalinityDoublerTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='salinity_doubler',
description='create a salinity doubler data product',
module='ion.processes.data.transforms.example_double_salinity',
class_name='SalinityDoubler',
process_source='SalinityDoubler source code here...')
try:
salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new SalinityDoubler data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(salinity_double_stream_def_id, salinity_doubler_dprocdef_id )
# Create the output data product of the transform
log.debug("create output data product SalinityDoubler")
salinity_doubler_output_dp_obj = IonObject(RT.DataProduct, name='SalinityDoubler',description='transform output salinity doubler')
salinity_doubler_output_dp_id = self.dataproductclient.create_data_product(salinity_doubler_output_dp_obj, salinity_double_stream_def_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=salinity_doubler_output_dp_id, persist_data=True, persist_metadata=True)
# Create the Salinity transform data process
log.debug("create L2_salinity data_process and start it")
try:
salinity_double_data_process_id = self.dataprocessclient.create_data_process(salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id, {'output':salinity_doubler_output_dp_id})
self.dataprocessclient.activate_data_process(salinity_double_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
###
### Start the process for producing the CTD data
###
# process definition for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':'ion.processes.data.ctd_stream_publisher',
'class':'SimpleCtdPublisher'
}
ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(process_definition=producer_definition)
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':ctd_stream_id,
}
}
ctd_sim_pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration)
## get the stream id for the transform outputs
stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
sal_stream_id = stream_ids[0]
stream_ids, _ = self.rrclient.find_objects(salinity_doubler_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
sal_dbl_stream_id = stream_ids[0]
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery([ctd_stream_id, sal_stream_id,sal_dbl_stream_id]),
exchange_name = 'salinity_test',
name = "test salinity subscription",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn(' data received!')
results.append(message)
if len(results) >15:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='salinity_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=20))
#Stop the transform process
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
first_salinity_values = None
for message in results:
try:
psd = PointSupplementStreamParser(stream_definition=ctd_stream_def, stream_granule=message)
temp = psd.get_values('temperature')
print psd.list_field_names()
except KeyError as ke:
temp = None
if temp is not None:
assertions(isinstance(temp, numpy.ndarray))
print 'temperature=' + str(numpy.nanmin(temp))
first_salinity_values = None
else:
psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message)
print psd.list_field_names()
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
print 'salinity=' + str(numpy.nanmin(salinity))
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
if first_salinity_values is None:
first_salinity_values = salinity.tolist()
else:
second_salinity_values = salinity.tolist()
assertions(len(first_salinity_values) == len(second_salinity_values))
for idx in range(0,len(first_salinity_values)):
assertions(first_salinity_values[idx]*2.0 == second_salinity_values[idx])
def instrument_test_driver(container):
org_client = OrgManagementServiceClient(node=container.node)
id_client = IdentityManagementServiceClient(node=container.node)
system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor)
log.info("system actor:" + system_actor._id)
sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id))
# Names of agent data streams to be configured.
parsed_stream_name = "ctd_parsed"
raw_stream_name = "ctd_raw"
# Driver configuration.
# Simulator
driver_config = {
"svr_addr": "localhost",
"cmd_port": 5556,
"evt_port": 5557,
"dvr_mod": "ion.services.mi.drivers.sbe37_driver",
"dvr_cls": "SBE37Driver",
"comms_config": {
SBE37Channel.CTD: {
"method": "ethernet",
"device_addr": CFG.device.sbe37.host,
"device_port": CFG.device.sbe37.port,
"server_addr": "localhost",
"server_port": 8888,
}
},
}
# Hardware
_container_client = ContainerAgentClient(node=container.node, name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info("Subscriber received message: %s", str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=container, node=container.node)
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = _pubsub_client.create_stream_definition(container=stream_def)
stream_id = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding="ION R2",
headers={"ion-actor-id": system_actor._id, "ion-actor-roles": sa_header_roles},
)
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = "%s_queue" % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = _pubsub_client.create_subscription(query=query, exchange_name=exchange_name)
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = "123xyz"
agent_config = {
"driver_config": driver_config,
"stream_config": stream_config,
"agent": {"resource_id": agent_resource_id},
}
# Launch an instrument agent process.
_ia_name = "agent007"
_ia_mod = "ion.services.mi.instrument_agent"
_ia_class = "InstrumentAgent"
_ia_pid = _container_client.spawn_process(name=_ia_name, module=_ia_mod, cls=_ia_class, config=agent_config)
log.info("got pid=%s for resource_id=%s" % (str(_ia_pid), str(agent_resource_id)))
def test_blog_ingestion_replay(self):
#-----------------------------------------------------------------------------------------------------
# Do this statement just once in your script
#-----------------------------------------------------------------------------------------------------
cc = self.container
#-------------------------------------------------------------------------------------------------------
# Make a registrar object - this is work usually done for you by the container in a transform or data stream process
#-------------------------------------------------------------------------------------------------------
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
#-----------------------------------------------------------------------------------------------------
# Service clients
#-----------------------------------------------------------------------------------------------------
ingestion_cli = IngestionManagementServiceClient(node=cc.node)
dr_cli = DataRetrieverServiceClient(node=cc.node)
dsm_cli = DatasetManagementServiceClient(node=cc.node)
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
#-------------------------------------------------------------------------------------------------------
# Create and activate ingestion configuration
#-------------------------------------------------------------------------------------------------------
ingestion_configuration_id = ingestion_cli.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name='dm_datastore',datastore_profile='EXAMPLES'),
hdf_storage=HdfStorage(),
number_of_workers=6,
)
# activates the transforms... so bindings will be created in this step
ingestion_cli.activate_ingestion_configuration(ingestion_configuration_id)
#------------------------------------------------------------------------------------------------------
# Create subscriber to listen to the messages published to the ingestion
#------------------------------------------------------------------------------------------------------
# Define the query we want
query = ExchangeQuery()
# Create the stateful listener to hold the captured data for comparison with replay
captured_input = BlogListener()
# Make a subscription to the input stream to ingestion
subscription_id = pubsub_cli.create_subscription(query = query, exchange_name='input_capture_queue' ,name = 'input_capture_queue')
# It is not required or even generally a good idea to use the subscription resource name as the queue name, but it makes things simple here
# Normally the container creates and starts subscribers for you when a transform process is spawned
subscriber = subscriber_registrar.create_subscriber(exchange_name='input_capture_queue', callback=captured_input.blog_store)
subscriber.start()
captured_input.subscriber = subscriber
pubsub_cli.activate_subscription(subscription_id)
#-------------------------------------------------------------------------------------------------------
# Launching blog scraper
#-------------------------------------------------------------------------------------------------------
blogs = [
'saintsandspinners',
'strobist',
'voodoofunk'
]
log.debug('before spawning blog scraper')
for blog in blogs:
config = {'process':{'type':'stream_process','blog':blog}}
cc.spawn_process(name=blog,
module='ion.services.dm.ingestion.example.blog_scraper',
cls='FeedStreamer',
config=config)
# wait ten seconds for some data to come in...
log.warn('Sleeping for 10 seconds to wait for some input')
time.sleep(10)
#------------------------------------------------------------------------------------------------------
# For 3 posts captured, make 3 replays and verify we get back what came in
#------------------------------------------------------------------------------------------------------
# Cute list comprehension method does not give enough control
#self.assertTrue(len(captured_input.blogs)>3)
#post_ids = [id for idx, id in enumerate(captured_input.blogs.iterkeys()) if idx < 3]
post_ids = []
for post_id, blog in captured_input.blogs.iteritems(): # Use items not iter items - I copy of fixed length
log.info('Captured Input: %s' % post_id)
if len(blog.get('comments',[])) > 2:
post_ids.append(post_id)
if len(post_ids) >3:
break
###=======================================================
### This section is not scriptable
###=======================================================
if len(post_ids) < 3:
self.fail('Not enough comments returned by the blog scrappers in 30 seconds')
if len(captured_input.blogs) < 1:
self.fail('No data returned in ten seconds by the blog scrappers!')
###=======================================================
### End non-scriptable
###=======================================================
#------------------------------------------------------------------------------------------------------
# Create subscriber to listen to the replays
#------------------------------------------------------------------------------------------------------
captured_replays = {}
for idx, post_id in enumerate(post_ids):
# Create the stateful listener to hold the captured data for comparison with replay
dataset_id = dsm_cli.create_dataset(
stream_id=post_id,
datastore_name='dm_datastore',
view_name='posts/posts_join_comments')
replay_id, stream_id =dr_cli.define_replay(dataset_id)
query = StreamQuery(stream_ids=[stream_id])
captured_replay = BlogListener()
#------------------------------------------------------------------------------------------------------
# Create subscriber to listen to the messages published to the ingestion
#------------------------------------------------------------------------------------------------------
# Make a subscription to the input stream to ingestion
subscription_name = 'replay_capture_queue_%d' % idx
subscription_id = pubsub_cli.create_subscription(query = query, exchange_name=subscription_name ,name = subscription_name)
# It is not required or even generally a good idea to use the subscription resource name as the queue name, but it makes things simple here
# Normally the container creates and starts subscribers for you when a transform process is spawned
subscriber = subscriber_registrar.create_subscriber(exchange_name=subscription_name, callback=captured_replay.blog_store)
subscriber.start()
captured_replay.subscriber = subscriber
pubsub_cli.activate_subscription(subscription_id)
#------------------------------------------------------------------------------------------------------
# Start the replay and listen to the results!
#------------------------------------------------------------------------------------------------------
dr_cli.start_replay(replay_id)
captured_replays[post_id] = captured_replay
###=======================================================
### The rest is not scriptable
###=======================================================
# wait five seconds for some data to come in...
log.warn('Sleeping for 5 seconds to wait for some output')
time.sleep(5)
matched_comments={}
for post_id, captured_replay in captured_replays.iteritems():
# There should be only one blog in here!
self.assertEqual(len(captured_replay.blogs),1)
replayed_blog = captured_replay.blogs[post_id]
input_blog = captured_input.blogs[post_id]
self.assertEqual(replayed_blog['post'].content, input_blog['post'].content)
# can't deterministically assert that the number of comments is the same...
matched_comments[post_id] = 0
for updated, comment in replayed_blog.get('comments',{}).iteritems():
self.assertIn(updated, input_blog['comments'])
matched_comments[post_id] += 1
# Assert that we got some comments back!
self.assertTrue(sum(matched_comments.values()) > 0)
log.info('Matched comments on the following blogs: %s' % matched_comments)
def test_activateInstrumentStream(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'new InstrumentModel id = ', instModel_id
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentStream: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_activateInstrumentStream: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
driver_config = {
'dvr_mod' : 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls' : 'SBE37Driver',
'workdir' : '/tmp/',
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug( 'test_activateInstrumentStream new Stream Definition id = %s', instDevice_id )
log.debug( 'test_activateInstrumentStream Creating new CDM data product with a stream definition' )
dp_obj = IonObject(RT.DataProduct,name='the parsed data',description='ctd stream test')
try:
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
log.debug( 'test_activateInstrumentStream new dp_id = %s', str(data_product_id1) )
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'test_activateInstrumentStream Data product streams1 = %s', str(stream_ids) )
simdata_subscription_id = self.pubsubcli.create_subscription(
query=StreamQuery([stream_ids[0]]),
exchange_name='Sim_data_queue',
name='SimDataSubscription',
description='SimData SubscriptionDescription'
)
def simdata_message_received(message, headers):
input = str(message)
log.debug("test_activateInstrumentStream: granule received: %s", input)
subscriber_registrar = StreamSubscriberRegistrar(process=self.container, node=self.container.node)
simdata_subscriber = subscriber_registrar.create_subscriber(exchange_name='Sim_data_queue', callback=simdata_message_received)
# Start subscribers
simdata_subscriber.start()
# Activate subscriptions
self.pubsubcli.activate_subscription(simdata_subscription_id)
log.debug( 'test_activateInstrumentStream Creating new RAW data product with a stream definition' )
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,name='the raw data',description='raw stream test')
try:
data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
log.debug( 'test_activateInstrumentStream new dp_id = %s', str(data_product_id2) )
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug( 'test_activateInstrumentStream Data product streams2 = %s', str(stream_ids) )
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
log.debug( 'test_activateInstrumentStream Instrument agent instance obj: = %s', str(inst_agent_instance_obj) )
# Start a resource agent client to talk with the instrument agent.
#self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess())
log.debug( 'test_activateInstrumentStream: got ia client %s', str(self._ia_client ))
log.debug("test_activateInstrumentStream: got ia client %s", str(self._ia_client))
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: initialize %s", str(retval))
time.sleep(2)
log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state))
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrumentStream: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentStream: return from go_streaming %s", str(reply))
time.sleep(15)
log.debug("test_activateInstrumentStream: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentStream: return from go_observatory %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrumentStream: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return from reset %s", str(reply))
time.sleep(2)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
def on_start(self):
rr_cli = ResourceRegistryServiceProcessClient(process=self,
node=self.container.node)
pubsub_cli = PubsubManagementServiceProcessClient(
process=self, node=self.container.node)
# Get the stream(s)
data_product_id = self.CFG.get_safe('dispatcher.data_product_id', '')
stream_ids, _ = rr_cli.find_objects(subject=data_product_id,
predicate=PRED.hasStream,
id_only=True)
log.info('Got Stream Ids: "%s"', stream_ids)
assert stream_ids, 'No streams found for this data product!'
query = StreamQuery(stream_ids=stream_ids)
exchange_name = 'dispatcher_%s' % str(os.getpid())
subscription_id = pubsub_cli.create_subscription(
query=query,
exchange_name=exchange_name,
name="SampleSubscription",
description="Sample Subscription Description")
stream_subscriber = StreamSubscriberRegistrar(process=self,
node=self.container.node)
stream_defs = {}
def message_received(granule, h):
stream_id = granule.stream_resource_id
data_stream_id = granule.data_stream_id
data_stream = granule.identifiables[data_stream_id]
tstamp = get_datetime(data_stream.timestamp.value)
records = granule.identifiables['record_count'].value
log.info(
'Received a message from stream %s with time stamp %s and %d records'
% (stream_id, tstamp, records))
if stream_id not in stream_defs:
stream_defs[stream_id] = pubsub_cli.find_stream_definition(
stream_id, id_only=False).container
stream_def = stream_defs.get(stream_id)
sp = PointSupplementStreamParser(stream_definition=stream_def,
stream_granule=granule)
last_data = {}
for field in sp.list_field_names():
last_data[field] = sp.get_values(field)[-1]
log.info('Last values in the message: %s' % str(last_data))
subscriber = stream_subscriber.create_subscriber(
exchange_name=exchange_name, callback=message_received)
subscriber.start()
pubsub_cli.activate_subscription(subscription_id)
class PubSubIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1",
description="Sample Stream 1 Description")
self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2",
description="Sample Stream 2 Description")
# Make a subscription to two input streams
exchange_name = "a_queue"
query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id])
self.ctd_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleSubscription",
"Sample Subscription Description")
# Make a subscription to all streams on an exchange point
exchange_name = "another_queue"
query = ExchangeQuery()
self.exchange_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleExchangeSubscription",
"Sample Exchange Subscription Description")
pid = self.container.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = self.container.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.container.node)
self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream1_id)
self.ctd_stream2_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream2_id)
# Cheat and use the cc as the process - I don't think it is used for anything...
self.stream_subscriber = StreamSubscriberRegistrar(process=dummy_process, node=self.container.node)
def tearDown(self):
self.pubsub_cli.delete_subscription(self.ctd_subscription_id)
self.pubsub_cli.delete_subscription(self.exchange_subscription_id)
self.pubsub_cli.delete_stream(self.ctd_stream1_id)
self.pubsub_cli.delete_stream(self.ctd_stream2_id)
self._stop_container()
def test_bind_stream_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='a_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.ctd_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(q.get(timeout=5), 'message2')
self.assertTrue(q.empty())
subscriber.stop()
def test_bind_exchange_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='another_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.exchange_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(q.get(timeout=5), 'message2')
self.assertTrue(q.empty())
subscriber.stop()
def test_unbind_stream_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='a_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.ctd_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.pubsub_cli.deactivate_subscription(self.ctd_subscription_id)
self.ctd_stream2_publisher.publish('message2')
p = None
with self.assertRaises(gevent.queue.Empty) as cm:
p = q.get(timeout=1)
subscriber.stop()
ex = cm.exception
self.assertEqual(str(ex), '')
self.assertEqual(p, None)
def test_unbind_exchange_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='another_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.exchange_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.pubsub_cli.deactivate_subscription(self.exchange_subscription_id)
self.ctd_stream2_publisher.publish('message2')
p = None
with self.assertRaises(gevent.queue.Empty) as cm:
p = q.get(timeout=1)
subscriber.stop()
ex = cm.exception
self.assertEqual(str(ex), '')
self.assertEqual(p, None)
def test_update_stream_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='a_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.ctd_subscription_id)
# Both publishers are received by the subscriber
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(q.get(timeout=5), 'message2')
self.assertTrue(q.empty())
# Update the subscription by removing a stream...
subscription = self.pubsub_cli.read_subscription(self.ctd_subscription_id)
stream_ids = list(subscription.query.stream_ids)
stream_ids.remove(self.ctd_stream2_id)
self.pubsub_cli.update_subscription(
subscription_id=subscription._id,
query=StreamQuery(stream_ids=stream_ids)
)
# Stream 2 is no longer received
self.ctd_stream2_publisher.publish('message2')
p = None
with self.assertRaises(gevent.queue.Empty) as cm:
p = q.get(timeout=1)
ex = cm.exception
self.assertEqual(str(ex), '')
self.assertEqual(p, None)
# Stream 1 is as before
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
# Now swith the active streams...
# Update the subscription by removing a stream...
self.pubsub_cli.update_subscription(
subscription_id=self.ctd_subscription_id,
query=StreamQuery([self.ctd_stream2_id])
)
# Stream 1 is no longer received
self.ctd_stream1_publisher.publish('message1')
p = None
with self.assertRaises(gevent.queue.Empty) as cm:
p = q.get(timeout=1)
ex = cm.exception
self.assertEqual(str(ex), '')
self.assertEqual(p, None)
# Stream 2 is received
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(q.get(timeout=5), 'message2')
self.assertTrue(q.empty())
subscriber.stop()
def test_find_stream_definition(self):
definition = SBE37_CDM_stream_definition()
definition_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=definition_id)
res_id = self.pubsub_cli.find_stream_definition(stream_id=stream_id, id_only=True)
self.assertTrue(res_id==definition_id, 'The returned id did not match the definition_id')
res_obj = self.pubsub_cli.find_stream_definition(stream_id=stream_id, id_only=False)
self.assertTrue(isinstance(res_obj.container, StreamDefinitionContainer),
'The container object is not a stream definition.')
def test_strem_def_not_found(self):
with self.assertRaises(NotFound):
self.pubsub_cli.find_stream_definition(stream_id='nonexistent')
definition = SBE37_CDM_stream_definition()
definition_id = self.pubsub_cli.create_stream_definition(container=definition)
with self.assertRaises(NotFound):
self.pubsub_cli.find_stream_definition(stream_id='nonexistent')
stream_id = self.pubsub_cli.create_stream()
with self.assertRaises(NotFound):
self.pubsub_cli.find_stream_definition(stream_id=stream_id)
@unittest.skip("Nothing to test")
def test_bind_already_bound_subscription(self):
pass
@unittest.skip("Nothing to test")
def test_unbind_unbound_subscription(self):
pass
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
def on_start(self):
# The data dictionary object holds a copy of all the viz products created by the service. The viz
# products are indexed by the viz_product_type and data_product_id (which could be google_datatables or
# mpl_graphs
self.viz_data_dictionary = {}
self.viz_data_dictionary['google_dt'] = {}
self.viz_data_dictionary['google_realtime_dt'] = {}
self.viz_data_dictionary['matplotlib_graphs'] = {}
# Kind of redundant but we will maintain a separate list of data product_ids registered with the viz_service
self.data_products = []
# Create clients to interface with PubSub, Transform Management Service and Resource Registry
self.pubsub_cli = self.clients.pubsub_management
self.tms_cli = self.clients.transform_management
self.rr_cli = self.clients.resource_registry
self.dr_cli = self.clients.data_retriever
self.dsm_cli = self.clients.dataset_management
"""
# Create process definitions which will used to spawn off the transform processes
self.matplotlib_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator())
self.matplotlib_proc_def.executable = {
'module': 'ion.services.ans.visualization_service',
'class':'VizTransformProcForMatplotlibGraphs'
}
self.matplotlib_proc_def_id, _ = self.rr_cli.create(self.matplotlib_proc_def)
self.google_dt_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator())
self.google_dt_proc_def.executable = {
'module': 'ion.services.ans.visualization_service',
'class':'VizTransformProcForGoogleDT'
}
self.google_dt_proc_def_id, _ = self.rr_cli.create(self.google_dt_proc_def)
"""
# Query resource registry to get process definitions and streams ids made by the bootstrap
proc_def_ids, _ = self.rr_cli.find_resources(
restype=RT.ProcessDefinition,
lcstate=None,
name="viz_matplotlib_transform_process",
id_only=True)
self.matplotlib_proc_def_id = proc_def_ids[0]
proc_def_ids, _ = self.rr_cli.find_resources(
restype=RT.ProcessDefinition,
lcstate=None,
name="viz_google_dt_transform_process",
id_only=True)
self.google_dt_proc_def_id = proc_def_ids[0]
# Create a stream that all the transform processes will use to submit data back to the viz service
self.viz_service_submit_stream_id = self.pubsub_cli.create_stream(
name="visualization_service_submit_stream." +
self.random_id_generator())
# subscribe to this stream since all the results from transforms will be submitted here
query = StreamQuery(stream_ids=[
self.viz_service_submit_stream_id,
])
self.viz_service_submit_stream_sub_id = self.pubsub_cli.create_subscription(
query=query, exchange_name="visualization_service_submit_queue")
submit_stream_subscriber_registrar = StreamSubscriberRegistrar(
process=self.container, node=self.container.node)
submit_stream_subscriber = submit_stream_subscriber_registrar.create_subscriber(
exchange_name='visualization_service_submit_queue',
callback=self.process_submission)
submit_stream_subscriber.start()
self.pubsub_cli.activate_subscription(
self.viz_service_submit_stream_sub_id)
# Discover the existing data_product_ids active in the system
sys_prod_ids, _ = self.rr_cli.find_resources(RT.DataProduct, None,
None, True)
# Register all the streams in the system, which will in turn start transform processes
for dp_id in sys_prod_ids:
self.register_new_data_product(dp_id)
# listen for events when new data_products show up
self.event_subscriber = EventSubscriber(
event_type="ResourceModifiedEvent",
origin_type="DataProduct",
sub_type="UPDATE",
callback=self.receive_new_dataproduct_event)
self.event_subscriber.activate()
return
def test_raw_stream_integration(self):
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here to run as a script (don't forget the imports of course!)
#-----------------------------
# Create some service clients...
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(
node=cc.node)
dataset_management_service = DatasetManagementServiceClient(
node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
# declare some handy variables
datastore_name = 'test_dm_integration'
###
### In the beginning there was one stream definitions...
###
# create a stream definition for the data from the ctd simulator
raw_ctd_stream_def = SBE37_RAW_stream_definition()
raw_ctd_stream_def_id = pubsub_management_service.create_stream_definition(
container=raw_ctd_stream_def, name='Simulated RAW CTD data')
###
### And two process definitions...
###
# one for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.raw_stream_publisher',
'class': 'RawStreamPublisher'
}
raw_ctd_sim_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
#---------------------------
# Set up ingestion - this is an operator concern - not done by SA in a deployed system
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,
datastore_profile='SCIDATA'),
number_of_workers=1)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
#---------------------------
# Set up the producer (CTD Simulator)
#---------------------------
# Create the stream
raw_ctd_stream_id = pubsub_management_service.create_stream(
stream_definition_id=raw_ctd_stream_def_id)
# Set up the datasets
raw_ctd_dataset_id = dataset_management_service.create_dataset(
stream_id=raw_ctd_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of this dataset
raw_ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=raw_ctd_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
# Start the ctd simulator to produce some data
configuration = {
'process': {
'stream_id': raw_ctd_stream_id,
}
}
raw_sim_pid = process_dispatcher.schedule_process(
process_definition_id=raw_ctd_sim_procdef_id,
configuration=configuration)
###
### Make a subscriber in the test to listen for salinity data
###
raw_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery([
raw_ctd_stream_id,
]),
exchange_name='raw_test',
name="test raw subscription",
)
# this is okay - even in cei mode!
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process,
node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn('Raw data received!')
results.append(message)
if len(results) > 3:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(
exchange_name='raw_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
pubsub_management_service.activate_subscription(
subscription_id=raw_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=10))
# stop the flow parse the messages...
process_dispatcher.cancel_process(
raw_sim_pid
) # kill the ctd simulator process - that is enough data
gevent.sleep(1)
for message in results:
sha1 = message.identifiables['stream_encoding'].sha1
data = message.identifiables['data_stream'].values
filename = FileSystem.get_hierarchical_url(FS.CACHE, sha1, ".raw")
with open(filename, 'r') as f:
assertions(data == f.read())
class PubSubIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.cc = ContainerAgentClient(node=self.container.node,name=self.container.name)
self.cc.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.cc.node)
self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1",
description="Sample Stream 1 Description")
self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2",
description="Sample Stream 2 Description")
# Make a subscription to two input streams
exchange_name = "a_queue"
query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id])
self.ctd_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleSubscription",
"Sample Subscription Description")
# Make a subscription to all streams on an exchange point
exchange_name = "another_queue"
query = ExchangeQuery()
self.exchange_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleExchangeSubscription",
"Sample Exchange Subscription Description")
pid = self.container.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = self.container.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.cc.node)
self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream1_id)
self.ctd_stream2_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream2_id)
# Cheat and use the cc as the process - I don't think it is used for anything...
self.stream_subscriber = StreamSubscriberRegistrar(process=dummy_process, node=self.cc.node)
def tearDown(self):
self.pubsub_cli.delete_subscription(self.ctd_subscription_id)
self.pubsub_cli.delete_subscription(self.exchange_subscription_id)
self.pubsub_cli.delete_stream(self.ctd_stream1_id)
self.pubsub_cli.delete_stream(self.ctd_stream2_id)
self._stop_container()
def test_bind_stream_subscription(self):
ar = gevent.event.AsyncResult()
self.first = True
def message_received(message, headers):
ar.set(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='a_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.ctd_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(ar.get(timeout=30), 'message1')
ar = gevent.event.AsyncResult()
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(ar.get(timeout=10), 'message2')
subscriber.stop()
def test_bind_exchange_subscription(self):
ar = gevent.event.AsyncResult()
self.first = True
def message_received(message, headers):
ar.set(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='another_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.exchange_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(ar.get(timeout=10), 'message1')
ar = gevent.event.AsyncResult()
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(ar.get(timeout=10), 'message2')
subscriber.stop()
def test_unbind_stream_subscription(self):
ar = gevent.event.AsyncResult()
self.first = True
def message_received(message, headers):
ar.set(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='a_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.ctd_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(ar.get(timeout=10), 'message1')
self.pubsub_cli.deactivate_subscription(self.ctd_subscription_id)
ar = gevent.event.AsyncResult()
self.ctd_stream2_publisher.publish('message2')
p = None
with self.assertRaises(gevent.Timeout) as cm:
p = ar.get(timeout=2)
subscriber.stop()
ex = cm.exception
self.assertEqual(str(ex), '2 seconds')
self.assertEqual(p, None)
def test_unbind_exchange_subscription(self):
ar = gevent.event.AsyncResult()
self.first = True
def message_received(message, headers):
ar.set(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='another_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.exchange_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(ar.get(timeout=10), 'message1')
self.pubsub_cli.deactivate_subscription(self.exchange_subscription_id)
ar = gevent.event.AsyncResult()
self.ctd_stream2_publisher.publish('message2')
p = None
with self.assertRaises(gevent.Timeout) as cm:
p = ar.get(timeout=2)
subscriber.stop()
ex = cm.exception
self.assertEqual(str(ex), '2 seconds')
self.assertEqual(p, None)
@unittest.skip("Nothing to test")
def test_bind_already_bound_subscription(self):
pass
@unittest.skip("Nothing to test")
def test_unbind_unbound_subscription(self):
pass
def setUp(self):
"""
Setup the test environment to exersice use of instrumet agent, including:
* define driver_config parameters.
* create container with required services and container client.
* create publication stream ids for each driver data stream.
* create stream_config parameters.
* create and activate subscriptions for agent data streams.
* spawn instrument agent process and create agent client.
* add cleanup functions to cause subscribers to get stopped.
"""
# params = { ('CTD', 'TA2'): -1.9434316e-05,
# ('CTD', 'PTCA1'): 1.3206866,
# ('CTD', 'TCALDATE'): [8, 11, 2006] }
# for tup in params:
# print tup
self.addCleanup(self.customCleanUp)
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method': 'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
'''
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': '137.110.112.119',
'device_port': 4001,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
'''
# Start container.
self._start_container()
# Establish endpoint with container (used in tests below)
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
# Create a pubsub client to create streams.
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(
process=self.container, node=self.container.node)
self.subs = []
# Create streams for each stream named in driver.
self.stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = self._pubsub_client.create_stream_definition(
container=stream_def)
stream_id = self._pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self.stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(
exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = self._pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name)
self._pubsub_client.activate_subscription(sub_id)
self.subs.append(sub)
# Add cleanup function to stop subscribers.
def stop_subscriber(sub_list):
for sub in sub_list:
sub.stop()
self.addCleanup(stop_subscriber, self.subs)
# Create agent config.
self.agent_resource_id = '123xyz'
self.agent_config = {
'driver_config': self.driver_config,
'stream_config': self.stream_config,
'agent': {
'resource_id': self.agent_resource_id
}
}
# Launch an instrument agent process.
self._ia_name = 'agent007'
self._ia_mod = 'ion.agents.instrument.instrument_agent'
self._ia_class = 'InstrumentAgent'
self._ia_pid = self._container_client.spawn_process(
name=self._ia_name,
module=self._ia_mod,
cls=self._ia_class,
config=self.agent_config)
log.info('got pid=%s', str(self._ia_pid))
self._ia_client = None
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(self.agent_resource_id,
process=FakeProcess())
log.info('got ia client %s', str(self._ia_client))
def test_dm_integration(self):
'''
test_salinity_transform
Test full DM Services Integration
'''
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here to run as a script (don't forget the imports of course!)
#-----------------------------
# Create some service clients...
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(node=cc.node)
dataset_management_service = DatasetManagementServiceClient(node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
transform_management_service = TransformManagementServiceClient(node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
# declare some handy variables
datastore_name = 'test_dm_integration'
###
### In the beginning there were two stream definitions...
###
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = pubsub_management_service.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data')
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = pubsub_management_service.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='Scalar Salinity data stream')
###
### And two process definitions...
###
# one for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':'ion.processes.data.ctd_stream_publisher',
'class':'SimpleCtdPublisher'
}
ctd_sim_procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
# one for the salinity transform
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':'ion.processes.data.transforms.ctd.ctd_L2_salinity',
'class':'SalinityTransform'
}
salinity_transform_procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
#---------------------------
# Set up ingestion - this is an operator concern - not done by SA in a deployed system
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,datastore_profile='SCIDATA'),
number_of_workers=1
)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
#---------------------------
# Set up the producer (CTD Simulator)
#---------------------------
# Create the stream
ctd_stream_id = pubsub_management_service.create_stream(stream_definition_id=ctd_stream_def_id)
# Set up the datasets
ctd_dataset_id = dataset_management_service.create_dataset(
stream_id=ctd_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule'
)
# Configure ingestion of this dataset
ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id = ctd_dataset_id,
archive_data = True,
archive_metadata = True,
ingestion_configuration_id = ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
#---------------------------
# Set up the salinity transform
#---------------------------
# Create the stream
sal_stream_id = pubsub_management_service.create_stream(stream_definition_id=sal_stream_def_id)
# Set up the datasets
sal_dataset_id = dataset_management_service.create_dataset(
stream_id=sal_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule'
)
# Configure ingestion of the salinity as a dataset
sal_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id = sal_dataset_id,
archive_data = True,
archive_metadata = True,
ingestion_configuration_id = ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto sal_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
# Create a subscription as input to the transform
sal_transform_input_subscription_id = pubsub_management_service.create_subscription(
query = StreamQuery(stream_ids=[ctd_stream_id,]),
exchange_name='salinity_transform_input') # how do we make these names??? i.e. Should they be anonymous?
# create the salinity transform
sal_transform_id = transform_management_service.create_transform(
name='example salinity transform',
in_subscription_id=sal_transform_input_subscription_id,
out_streams={'output':sal_stream_id,},
process_definition_id = salinity_transform_procdef_id,
# no configuration needed at this time...
)
# start the transform - for a test case it makes sense to do it before starting the producer but it is not required
transform_management_service.activate_transform(transform_id=sal_transform_id)
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':ctd_stream_id,
}
}
ctd_sim_pid = process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration)
###
### Make a subscriber in the test to listen for salinity data
###
salinity_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery([sal_stream_id,]),
exchange_name = 'salinity_test',
name = "test salinity subscription",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn('Salinity data received!')
results.append(message)
if len(results) >3:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='salinity_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
pubsub_management_service.activate_subscription(subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=10))
# stop the flow parse the messages...
process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
for message in results:
psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message)
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
import numpy
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
def test_activateInstrumentStream(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
model_label="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
print 'new InstrumentModel id = ', instModel_id
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent")
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
print 'new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentStream: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
try:
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentStream: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
driver_config = {
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'workdir': '/tmp/',
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org',
comms_device_port=4001,
port_agent_work_dir='/tmp/',
port_agent_delimeter=['<<', '>>'])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
container=ctd_stream_def)
log.debug(
'test_activateInstrumentStream new Stream Definition id = %s',
instDevice_id)
log.debug(
'test_activateInstrumentStream Creating new CDM data product with a stream definition'
)
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test')
try:
data_product_id1 = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
log.debug('test_activateInstrumentStream new dp_id = %s',
str(data_product_id1))
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('test_activateInstrumentStream Data product streams1 = %s',
str(stream_ids))
simdata_subscription_id = self.pubsubcli.create_subscription(
query=StreamQuery([stream_ids[0]]),
exchange_name='Sim_data_queue',
name='SimDataSubscription',
description='SimData SubscriptionDescription')
def simdata_message_received(message, headers):
input = str(message)
log.debug("test_activateInstrumentStream: granule received: %s",
input)
subscriber_registrar = StreamSubscriberRegistrar(
process=self.container, node=self.container.node)
simdata_subscriber = subscriber_registrar.create_subscriber(
exchange_name='Sim_data_queue', callback=simdata_message_received)
# Start subscribers
simdata_subscriber.start()
# Activate subscriptions
self.pubsubcli.activate_subscription(simdata_subscription_id)
log.debug(
'test_activateInstrumentStream Creating new RAW data product with a stream definition'
)
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(
container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
try:
data_product_id2 = self.dpclient.create_data_product(
dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
log.debug('test_activateInstrumentStream new dp_id = %s',
str(data_product_id2))
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('test_activateInstrumentStream Data product streams2 = %s',
str(stream_ids))
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
log.debug(
'test_activateInstrumentStream Instrument agent instance obj: = %s',
str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
#self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id,
process=FakeProcess())
log.debug('test_activateInstrumentStream: got ia client %s',
str(self._ia_client))
log.debug("test_activateInstrumentStream: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: initialize %s", str(retval))
time.sleep(2)
log.debug(
"test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)"
)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug(
"test_activateInstrumentStream: return value from go_active %s",
str(reply))
time.sleep(2)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state))
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrumentStream: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentStream: return from go_streaming %s",
str(reply))
time.sleep(15)
log.debug("test_activateInstrumentStream: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute(cmd)
log.debug(
"test_activateInstrumentStream: return from go_observatory %s",
str(reply))
time.sleep(2)
log.debug("test_activateInstrumentStream: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return from reset %s",
str(reply))
time.sleep(2)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
def test_dm_integration(self):
'''
test_salinity_transform
Test full DM Services Integration
'''
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here to run as a script (don't forget the imports of course!)
#-----------------------------
# Create some service clients...
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(
node=cc.node)
dataset_management_service = DatasetManagementServiceClient(
node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
transform_management_service = TransformManagementServiceClient(
node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
# declare some handy variables
datastore_name = 'test_dm_integration'
###
### In the beginning there were two stream definitions...
###
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = pubsub_management_service.create_stream_definition(
container=ctd_stream_def, name='Simulated CTD data')
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = pubsub_management_service.create_stream_definition(
container=SalinityTransform.outgoing_stream_def,
name='Scalar Salinity data stream')
###
### And two process definitions...
###
# one for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.ctd_stream_publisher',
'class': 'SimpleCtdPublisher'
}
ctd_sim_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
# one for the salinity transform
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.transforms.ctd.ctd_L2_salinity',
'class': 'SalinityTransform'
}
salinity_transform_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
#---------------------------
# Set up ingestion - this is an operator concern - not done by SA in a deployed system
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,
datastore_profile='SCIDATA'),
number_of_workers=1)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
#---------------------------
# Set up the producer (CTD Simulator)
#---------------------------
# Create the stream
ctd_stream_id = pubsub_management_service.create_stream(
stream_definition_id=ctd_stream_def_id)
# Set up the datasets
ctd_dataset_id = dataset_management_service.create_dataset(
stream_id=ctd_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of this dataset
ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=ctd_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
#---------------------------
# Set up the salinity transform
#---------------------------
# Create the stream
sal_stream_id = pubsub_management_service.create_stream(
stream_definition_id=sal_stream_def_id)
# Set up the datasets
sal_dataset_id = dataset_management_service.create_dataset(
stream_id=sal_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of the salinity as a dataset
sal_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=sal_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto sal_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
# Create a subscription as input to the transform
sal_transform_input_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery(stream_ids=[
ctd_stream_id,
]),
exchange_name='salinity_transform_input'
) # how do we make these names??? i.e. Should they be anonymous?
# create the salinity transform
sal_transform_id = transform_management_service.create_transform(
name='example salinity transform',
in_subscription_id=sal_transform_input_subscription_id,
out_streams={
'output': sal_stream_id,
},
process_definition_id=salinity_transform_procdef_id,
# no configuration needed at this time...
)
# start the transform - for a test case it makes sense to do it before starting the producer but it is not required
transform_management_service.activate_transform(
transform_id=sal_transform_id)
# Start the ctd simulator to produce some data
configuration = {
'process': {
'stream_id': ctd_stream_id,
}
}
ctd_sim_pid = process_dispatcher.schedule_process(
process_definition_id=ctd_sim_procdef_id,
configuration=configuration)
###
### Make a subscriber in the test to listen for salinity data
###
salinity_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery([
sal_stream_id,
]),
exchange_name='salinity_test',
name="test salinity subscription",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process,
node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn('Salinity data received!')
results.append(message)
if len(results) > 3:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(
exchange_name='salinity_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
pubsub_management_service.activate_subscription(
subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=10))
# stop the flow parse the messages...
process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
for message in results:
psd = PointSupplementStreamParser(
stream_definition=SalinityTransform.outgoing_stream_def,
stream_granule=message)
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
import numpy
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) >
0.0) # salinity should always be greater than 0
def on_start(self):
# The data dictionary object holds a copy of all the viz products created by the service. The viz
# products are indexed by the viz_product_type and data_product_id (which could be google_datatables or
# mpl_graphs
self.viz_data_dictionary = {}
self.viz_data_dictionary['google_dt'] = {}
self.viz_data_dictionary['google_realtime_dt'] = {}
self.viz_data_dictionary['matplotlib_graphs'] = {}
# Kind of redundant but we will maintain a separate list of data product_ids registered with the viz_service
self.data_products = []
# Create clients to interface with PubSub, Transform Management Service and Resource Registry
self.pubsub_cli = self.clients.pubsub_management
self.tms_cli = self.clients.transform_management
self.rr_cli = self.clients.resource_registry
self.dr_cli = self.clients.data_retriever
self.dsm_cli = self.clients.dataset_management
"""
# Create process definitions which will used to spawn off the transform processes
self.matplotlib_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator())
self.matplotlib_proc_def.executable = {
'module': 'ion.services.ans.visualization_service',
'class':'VizTransformProcForMatplotlibGraphs'
}
self.matplotlib_proc_def_id, _ = self.rr_cli.create(self.matplotlib_proc_def)
self.google_dt_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator())
self.google_dt_proc_def.executable = {
'module': 'ion.services.ans.visualization_service',
'class':'VizTransformProcForGoogleDT'
}
self.google_dt_proc_def_id, _ = self.rr_cli.create(self.google_dt_proc_def)
"""
# Query resource registry to get process definitions and streams ids made by the bootstrap
proc_def_ids,_ = self.rr_cli.find_resources(restype=RT.ProcessDefinition, lcstate=None, name="viz_matplotlib_transform_process", id_only=True)
self.matplotlib_proc_def_id = proc_def_ids[0]
proc_def_ids,_ = self.rr_cli.find_resources(restype=RT.ProcessDefinition, lcstate=None, name="viz_google_dt_transform_process", id_only=True)
self.google_dt_proc_def_id = proc_def_ids[0]
# Create a stream that all the transform processes will use to submit data back to the viz service
self.viz_service_submit_stream_id = self.pubsub_cli.create_stream(name="visualization_service_submit_stream." + self.random_id_generator())
# subscribe to this stream since all the results from transforms will be submitted here
query = StreamQuery(stream_ids=[self.viz_service_submit_stream_id,])
self.viz_service_submit_stream_sub_id = self.pubsub_cli.create_subscription(query=query, exchange_name="visualization_service_submit_queue")
submit_stream_subscriber_registrar = StreamSubscriberRegistrar(process = self.container, node = self.container.node )
submit_stream_subscriber = submit_stream_subscriber_registrar.create_subscriber(exchange_name='visualization_service_submit_queue', callback=self.process_submission)
submit_stream_subscriber.start()
self.pubsub_cli.activate_subscription(self.viz_service_submit_stream_sub_id)
# Discover the existing data_product_ids active in the system
sys_prod_ids, _ = self.rr_cli.find_resources(RT.DataProduct, None, None, True)
# Register all the streams in the system, which will in turn start transform processes
for dp_id in sys_prod_ids:
self.register_new_data_product(dp_id)
# listen for events when new data_products show up
self.event_subscriber = EventSubscriber(
event_type = "ResourceModifiedEvent",
origin_type = "DataProduct",
sub_type="UPDATE",
callback=self.receive_new_dataproduct_event
)
self.event_subscriber.activate()
return
def setUp(self):
"""
Setup the test environment to exersice use of instrumet agent, including:
* define driver_config parameters.
* create container with required services and container client.
* create publication stream ids for each driver data stream.
* create stream_config parameters.
* create and activate subscriptions for agent data streams.
* spawn instrument agent process and create agent client.
* add cleanup functions to cause subscribers to get stopped.
"""
# params = { ('CTD', 'TA2'): -1.9434316e-05,
# ('CTD', 'PTCA1'): 1.3206866,
# ('CTD', 'TCALDATE'): [8, 11, 2006] }
# for tup in params:
# print tup
self.addCleanup(self.customCleanUp)
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.services.mi.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
'''
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.services.mi.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': '137.110.112.119',
'device_port': 4001,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
'''
# Start container.
self._start_container()
# Establish endpoint with container (used in tests below)
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
# Create a pubsub client to create streams.
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
container=self.container)
self.subs = []
# Create streams for each stream named in driver.
self.stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = self._pubsub_client.create_stream_definition(
container=stream_def)
stream_id = self._pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self.stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = self._pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name)
self._pubsub_client.activate_subscription(sub_id)
self.subs.append(sub)
# Add cleanup function to stop subscribers.
def stop_subscriber(sub_list):
for sub in sub_list:
sub.stop()
self.addCleanup(stop_subscriber, self.subs)
# Create agent config.
self.agent_resource_id = '123xyz'
self.agent_config = {
'driver_config' : self.driver_config,
'stream_config' : self.stream_config,
'agent' : {'resource_id': self.agent_resource_id}
}
# Launch an instrument agent process.
self._ia_name = 'agent007'
self._ia_mod = 'ion.services.mi.instrument_agent'
self._ia_class = 'InstrumentAgent'
self._ia_pid = self._container_client.spawn_process(name=self._ia_name,
module=self._ia_mod, cls=self._ia_class,
config=self.agent_config)
log.info('got pid=%s', str(self._ia_pid))
self._ia_client = None
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(self.agent_resource_id, process=FakeProcess())
log.info('got ia client %s', str(self._ia_client))
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, container=cc)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, container=cc)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
def instrument_test_driver(container):
org_client = OrgManagementServiceClient(node=container.node)
id_client = IdentityManagementServiceClient(node=container.node)
system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor)
log.info('system actor:' + system_actor._id)
sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id))
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
_container_client = ContainerAgentClient(node=container.node,
name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=container,
node=container.node)
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = _pubsub_client.create_stream_definition(
container=stream_def)
stream_id = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': sa_header_roles })
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = _pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name )
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = '123xyz'
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': agent_resource_id}
}
# Launch an instrument agent process.
_ia_name = 'agent007'
_ia_mod = 'ion.agents.instrument.instrument_agent'
_ia_class = 'InstrumentAgent'
_ia_pid = _container_client.spawn_process(name=_ia_name,
module=_ia_mod, cls=_ia_class,
config=agent_config)
log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
def setUp(self):
"""
Setup the test environment to exersice use of instrumet agent, including:
* define driver_config parameters.
* create container with required services and container client.
* create publication stream ids for each driver data stream.
* create stream_config parameters.
* create and activate subscriptions for agent data streams.
* spawn instrument agent process and create agent client.
* add cleanup functions to cause subscribers to get stopped.
"""
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.services.mi.drivers.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': '137.110.112.119',
'device_port': 4001,
'server_addr': 'localhost',
'server_port': 8888
}
},
'packet_config' : {
parsed_stream_name : ('prototype.sci_data.ctd_stream',
'ctd_stream_packet'),
raw_stream_name : None
}
}
# Start container.
self._start_container()
# Establish endpoint with container.
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Bring up services in a deploy file.
self._container_client.start_rel_from_url('res/deploy/r2dm.yml')
# Create a pubsub client to create streams.
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# Create parsed stream. The stream name must match one
# used by the driver to label packet data.
parsed_stream_def = ctd_stream_definition(stream_id=None)
parsed_stream_def_id = self._pubsub_client.create_stream_definition(
container=parsed_stream_def)
parsed_stream_id = self._pubsub_client.create_stream(
name=parsed_stream_name,
stream_definition_id=parsed_stream_def_id,
original=True,
encoding='ION R2')
# Create raw stream. The stream name must match one used by the
# driver to label packet data. This stream does not yet have a
# packet definition so will not be published.
raw_stream_def = ctd_stream_definition(stream_id=None)
raw_stream_def_id = self._pubsub_client.create_stream_definition(
container=raw_stream_def)
raw_stream_id = self._pubsub_client.create_stream(name=raw_stream_name,
stream_definition_id=raw_stream_def_id,
original=True,
encoding='ION R2')
# Define stream configuration.
self.stream_config = {
parsed_stream_name : parsed_stream_id,
raw_stream_name : raw_stream_id
}
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
node=self.container.node)
# Create and activate parsed data subscription.
parsed_sub = subscriber_registrar.create_subscriber(exchange_name=\
'parsed_queue', callback=consume)
parsed_sub.start()
parsed_query = StreamQuery(stream_ids=[parsed_stream_id])
parsed_sub_id = self._pubsub_client.create_subscription(\
query=parsed_query, exchange_name='parsed_queue')
self._pubsub_client.activate_subscription(parsed_sub_id)
# Create and activate raw data subscription.
raw_sub = subscriber_registrar.create_subscriber(exchange_name=\
'raw_queue', callback=consume)
raw_sub.start()
raw_query = StreamQuery(stream_ids=[raw_stream_id])
raw_sub_id = self._pubsub_client.create_subscription(\
query=raw_query, exchange_name='raw_queue')
self._pubsub_client.activate_subscription(raw_sub_id)
# Create agent config.
self.agent_config = {
'driver_config' : self.driver_config,
'stream_config' : self.stream_config
}
# Launch an instrument agent process.
self._ia_name = 'agent007'
self._ia_mod = 'ion.services.mi.instrument_agent'
self._ia_class = 'InstrumentAgent'
self._ia_pid = self._container_client.spawn_process(name=self._ia_name,
module=self._ia_mod, cls=self._ia_class,
config=self.agent_config)
log.info('got pid=%s', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('123xyz', name=self._ia_pid,
process=FakeProcess())
log.info('got ia client %s', str(self._ia_client))
# Add cleanup function to stop subscribers.
def stop_subscriber(sub_list):
for sub in sub_list:
sub.stop()
self.addCleanup(stop_subscriber, [parsed_sub, raw_sub])
