class DemuxTransformIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub_client = PubsubManagementServiceClient()
self.queue_cleanup = list()
def tearDown(self):
for queue in self.queue_cleanup:
if isinstance(queue,ExchangeNameQueue):
queue.delete()
elif isinstance(queue,basestring):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
def test_demux(self):
self.stream0, self.route0 = self.pubsub_client.create_stream('stream0', exchange_point='test')
self.stream1, self.route1 = self.pubsub_client.create_stream('stream1', exchange_point='main_data')
self.stream2, self.route2 = self.pubsub_client.create_stream('stream2', exchange_point='alt_data')
self.r_stream1 = gevent.event.Event()
self.r_stream2 = gevent.event.Event()
def process(msg, stream_route, stream_id):
if stream_id == self.stream1:
self.r_stream1.set()
elif stream_id == self.stream2:
self.r_stream2.set()
self.container.spawn_process('demuxer', 'ion.processes.data.transforms.mux', 'DemuxTransform', {'process':{'out_streams':[self.stream1, self.stream2]}}, 'demuxer_pid')
self.queue_cleanup.append('demuxer_pid')
sub1 = StandaloneStreamSubscriber('sub1', process)
sub2 = StandaloneStreamSubscriber('sub2', process)
sub1.xn.bind(self.route1.routing_key, self.container.ex_manager.create_xp('main_data'))
sub2.xn.bind(self.route2.routing_key, self.container.ex_manager.create_xp('alt_data'))
sub1.start()
sub2.start()
self.queue_cleanup.append(sub1.xn)
self.queue_cleanup.append(sub2.xn)
xn = self.container.ex_manager.create_xn_queue('demuxer_pid')
xn.bind(self.route0.routing_key, self.container.ex_manager.create_xp(self.route0.exchange_point))
domino = StandaloneStreamPublisher(self.stream0, self.route0)
domino.publish('test')
self.assertTrue(self.r_stream1.wait(2))
self.assertTrue(self.r_stream2.wait(2))
self.container.proc_manager.terminate_process('demuxer_pid')
sub1.stop()
sub2.stop()
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = "parsed"
param_dict_name = "ctd_parsed_param_dict"
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id
)
stream_config = dict(
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict,
)
self._stream_config[stream_name] = stream_config
stream_name = "raw"
param_dict_name = "ctd_raw_param_dict"
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id
)
stream_config = dict(
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict,
)
self._stream_config[stream_name] = stream_config
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info("Subscriber received data message: %s.", str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container, node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name, stream_definition_id=stream_def_id, original=True, encoding="ION R2"
)
self._stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = "%s_queue" % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(query=query, exchange_name=exchange_name)
pubsub_client.activate_subscription(sub_id)
def build_stream_config(streams):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=cc.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
agent_stream_config = {}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
agent_stream_config[stream_name] = stream_config
return agent_stream_config
def _build_stream_config(self):
"""
"""
if (not self.packet_config):
return
streams = self.packet_config
log.debug("Streams: %s", streams)
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
for stream_name in streams:
pd_id = None
try:
pd_id = dataset_management.read_parameter_dictionary_by_name(
stream_name, id_only=True)
except:
log.error("No pd_id found for param_dict '%s'" % stream_name)
if (self.use_default_stream):
log.error("using default pd '%s'" % DEFAULT_STREAM_NAME)
pd_id = dataset_management.read_parameter_dictionary_by_name(
DEFAULT_STREAM_NAME, id_only=True)
if (not pd_id):
raise IDKException(
"Missing parameter dictionary for stream '%s'" %
stream_name)
log.debug("parameter dictionary id: %s" % pd_id)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
#log.debug("Stream: %s (%s), stream_def_id %s" % (stream_name, type(stream_name), stream_def_id))
pd = pubsub_client.read_stream_definition(
stream_def_id).parameter_dictionary
#log.debug("Parameter Dictionary: %s" % pd)
try:
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(
stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
#log.debug("Stream Config (%s): %s" % (stream_name, stream_config))
except Exception as e:
log.error("stream publisher exception: %s", e)
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {
'parsed' : 'ctd_parsed_param_dict',
'raw' : 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_PARAM_DICT, id_only=True)
if (not pd_id):
log.error("No pd_id found for param_dict '%s'" % DEFAULT_PARAM_DICT)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name,
parameter_dictionary_id=pd_id)
pd = None
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
def build_stream_config(streams):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=cc.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
agent_stream_config = {}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(
stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
agent_stream_config[stream_name] = stream_config
return agent_stream_config
def _build_stream_config(self):
"""
"""
if not self.packet_config:
return
streams = self.packet_config
log.debug("Streams: %s", streams)
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
for stream_name in streams:
pd_id = None
try:
pd_id = dataset_management.read_parameter_dictionary_by_name(stream_name, id_only=True)
except:
log.error("No pd_id found for param_dict '%s'" % stream_name)
if self.use_default_stream:
log.error("using default pd '%s'" % DEFAULT_STREAM_NAME)
pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_STREAM_NAME, id_only=True)
if not pd_id:
raise IDKException("Missing parameter dictionary for stream '%s'" % stream_name)
log.debug("parameter dictionary id: %s" % pd_id)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
# log.debug("Stream: %s (%s), stream_def_id %s" % (stream_name, type(stream_name), stream_def_id))
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
# log.debug("Parameter Dictionary: %s" % pd)
try:
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id
)
stream_config = dict(
stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd,
)
self.stream_config[stream_name] = stream_config
# log.debug("Stream Config (%s): %s" % (stream_name, stream_config))
except Exception as e:
log.error("stream publisher exception: %s", e)
log.debug("Stream config setup complete.")
def start_ctd_publisher(container):
pubsubclient = PubsubManagementServiceClient(node=container.node)
stream_id, route = pubsubclient.create_stream('ctd_publisher', exchange_point='science_data')
pid = container.spawn_process('ctdpublisher', 'ion.processes.data.sinusoidal_stream_publisher','SinusoidalCtdPublisher',{'process':{'stream_id':stream_id}})
print 'stream_id=' + stream_id
print 'pid=' + pid
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {
'parsed': 'ctd_parsed_param_dict',
'raw': 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(
stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(
stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
if stream_name == 'parsed':
type = 'IntervalAlarmDef'
kwargs = {
'name': 'test_sim_warning',
'stream_name': 'parsed',
'value_id': 'temp',
'message': 'Temperature is above test range of 5.0.',
'type': StreamAlarmType.WARNING,
'upper_bound': 5.0,
'upper_rel_op': '<'
}
alarm = {}
alarm['type'] = type
alarm['kwargs'] = kwargs
alarms = [alarm]
stream_config['alarms'] = alarms
self._stream_config[stream_name] = stream_config
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'parsed'
param_dict_name = 'ctd_parsed_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
stream_name = 'raw'
param_dict_name = 'ctd_raw_param_dict'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self._stream_config[stream_name] = stream_config
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
container=self.container)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
# TODO the following is a mininal adjustment to at least let the test
# continue:
# for (stream_name, val) in PACKET_CONFIG.iteritems():
for stream_name in PACKET_CONFIG:
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(
container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
taxy = get_taxonomy(stream_name)
stream_config = dict(
id=stream_id,
taxonomy=taxy.dump()
)
self._stream_config[stream_name] = stream_config
# self._stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(
query=query, exchange_name=exchange_name, exchange_point='science_data')
pubsub_client.activate_subscription(sub_id)
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {
'parsed' : 'ctd_parsed_param_dict',
'raw' : 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
if stream_name == 'parsed':
type = 'IntervalAlarmDef'
kwargs = {
'name' : 'test_sim_warning',
'stream_name' : 'parsed',
'value_id' : 'temp',
'message' : 'Temperature is above test range of 5.0.',
'type' : StreamAlarmType.WARNING,
'upper_bound' : 5.0,
'upper_rel_op' : '<'
}
alarm = {}
alarm['type'] = type
alarm['kwargs'] = kwargs
alarms = [alarm]
stream_config['alarms'] = alarms
self._stream_config[stream_name] = stream_config
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {"parsed": "ctd_parsed_param_dict", "raw": "ctd_raw_param_dict"}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id
)
stream_config = dict(
stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd,
)
if stream_name == "parsed":
type = "IntervalAlarmDef"
kwargs = {
"name": "test_sim_warning",
"stream_name": "parsed",
"value_id": "temp",
"message": "Temperature is above test range of 5.0.",
"type": StreamAlarmType.WARNING,
"upper_bound": 5.0,
"upper_rel_op": "<",
}
alarm = {}
alarm["type"] = type
alarm["kwargs"] = kwargs
alarms = [alarm]
stream_config["alarms"] = alarms
self._stream_config[stream_name] = stream_config
def run_external_transform(self):
'''
This example script illustrates how a transform can interact with the an outside process (very basic)
it launches an external_transform example which uses the operating system command 'bc' to add 1 to the input
Producer -> A -> 'FS.TEMP/transform_output'
A is an external transform that spawns an OS process to increment the input by 1
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = ProcessDefinition(name='external_transform_definition')
process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ExternalTransform'
process_definition_id = procd_cli.create_process_definition(process_definition=process_definition)
#-------------------------------
# Streams
#-------------------------------
input_stream_id = pubsub_cli.create_stream(name='input_stream', original=True)
#-------------------------------
# Subscription
#-------------------------------
query = StreamQuery(stream_ids=[input_stream_id])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
#-------------------------------
# Launch Transform
#-------------------------------
transform_id = tms_cli.create_transform(name='external_transform',
in_subscription_id=input_subscription_id,
process_definition_id=process_definition_id,
configuration={})
tms_cli.activate_transform(transform_id)
#-------------------------------
# Launch Producer
#-------------------------------
id_p = self.container.spawn_process('myproducer', 'ion.processes.data.transforms.transform_example', 'TransformExampleProducer', {'process':{'type':'stream_process','publish_streams':{'out_stream':input_stream_id}},'stream_producer':{'interval':4000}})
self.container.proc_manager.procs[id_p].start()
def __init__(self, packet_config = {}, stream_definition = None, original = True, encoding = "ION R2"):
log.info("Start data subscribers")
self.no_samples = None
self.async_data_result = AsyncResult()
self.data_greenlets = []
self.stream_config = {}
self.samples_received = []
self.data_subscribers = []
self.container = Container.instance
if not self.container:
raise NoContainer()
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self.samples_received.append(message)
if self.no_samples and self.no_samples == len(self.samples_received):
self.async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container, node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
self.data_subscribers = []
for (stream_name, val) in packet_config.iteritems():
stream_def_id = pubsub_client.create_stream_definition(container=stream_definition)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=original,
encoding=encoding)
self.stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume_data)
self._listen(sub)
self.data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(query=query, exchange_name=exchange_name)
pubsub_client.activate_subscription(sub_id)
class ScienceGranuleIngestionIntTest(IonIntegrationTestCase):
def setUp(self):
self.datastore_name = 'datasets'
self.exchange_point = 'science_data'
self.exchange_space = 'science_granule_ingestion'
self.queue_name = self.exchange_space
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.ingestion_management = IngestionManagementServiceClient()
self.pubsub = PubsubManagementServiceClient()
def build_granule(self):
return ScienceGranuleIngestionWorkerUnitTest.build_granule()
def launch_worker(self):
cfg = DotDict()
cfg.process.datastore_name = self.datastore_name
cfg.process.queue_name = self.queue_name
#@todo: replace with CEI friendly calls
pid = self.container.spawn_process('ingest_worker', 'ion.processes.data.ingestion.science_granule_ingestion_worker','ScienceGranuleIngestionWorker',cfg)
return pid
def create_ingestion_config(self):
ingest_queue = IngestionQueue(name=self.exchange_space, type='science_granule')
config_id = self.ingestion_management.create_ingestion_configuration(name='standard_ingest', exchange_point_id=self.exchange_point, queues=[ingest_queue])
return config_id
def create_stream(self):
stream_id = self.pubsub.create_stream()
return stream_id
def poll(self, evaluation_callback, *args, **kwargs):
now = time.time()
cutoff = now + 5
done = False
while not done:
if evaluation_callback(*args,**kwargs):
done = True
if now >= cutoff:
raise Timeout('No results found within the allotted time')
now = time.time()
return True
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(
self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(
process=self.container, container=self.container)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
for stream_name in PACKET_CONFIG:
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(
container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
taxy = get_taxonomy(stream_name)
stream_config = dict(id=stream_id, taxonomy=taxy.dump())
self._stream_config[stream_name] = stream_config
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(
exchange_name=exchange_name, callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(
query=query,
exchange_name=exchange_name,
exchange_point='science_data')
pubsub_client.activate_subscription(sub_id)
def on_start(self):
exchange_point = self.CFG.get_safe('process.exchange_point', 'science_data')
self.CFG.process.exchange_point = exchange_point
super(SimpleCtdPublisher,self).on_start()
self.stream_id = self.CFG.get_safe('process.stream_id',{})
self.interval = self.CFG.get_safe('process.interval', 1.0)
self.last_time = self.CFG.get_safe('process.last_time', 0)
# Stream creation is done in SA, but to make the example go for demonstration create one here if it is not provided...
if not self.stream_id:
pubsub_cli = PubsubManagementServiceClient()
self.stream_id = pubsub_cli.create_stream( name='Example CTD Data')
self.greenlet = gevent.spawn(self._trigger_func, self.stream_id)
self.finished = gevent.event.Event()
log.info('SimpleCTDPublisher started, publishing to %s->%s', self.exchange_point, self.stream_id)
def start_ctd_publisher(container):
pubsubclient = PubsubManagementServiceClient(node=container.node)
ctd_stream_def_id = pubsubclient.create_stream_definition(name='SBE37_CDM')
stream_id, route = pubsubclient.create_stream(
'ctd_publisher',
exchange_point='science_data',
stream_definition_id=ctd_stream_def_id)
pid = container.spawn_process(
'ctdpublisher', 'ion.processes.data.sinusoidal_stream_publisher',
'SinusoidalCtdPublisher', {'process': {
'stream_id': stream_id
}})
print 'stream_id=' + stream_id
print 'pid=' + pid
def on_start(self):
'''
Creates a publisher for each stream_id passed in as publish_streams
Creates an attribute with the name matching the stream name which corresponds to the publisher
ex: say we have publish_streams:{'output': my_output_stream_id }
then the instance has an attribute output which corresponds to the publisher for the stream
in my_output_stream_id
'''
# Get the stream(s)
stream_id = self.CFG.get('process',{}).get('stream_id','')
self.greenlet_queue = []
self._usgs_def = USGS_stream_definition()
# Stream creation is done in SA, but to make the example go for demonstration create one here if it is not provided...
if not stream_id:
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
stream_id = pubsub_cli.create_stream(
name='Example USGS Data',
stream_definition=self._usgs_def,
original=True,
encoding='ION R2')
self.stream_publisher_registrar = StreamPublisherRegistrar(process=self,node=self.container.node)
# Needed to get the originator's stream_id
self.stream_id = stream_id
self.publisher = self.stream_publisher_registrar.create_publisher(stream_id=stream_id)
self.last_time = 0
g = Greenlet(self._trigger_func, stream_id)
log.warn('Starting publisher thread for simple usgs data.')
g.start()
self.greenlet_queue.append(g)
def on_start(self):
'''
Creates a publisher for each stream_id passed in as publish_streams
Creates an attribute with the name matching the stream name which corresponds to the publisher
ex: say we have publish_streams:{'output': my_output_stream_id }
then the instance has an attribute output which corresponds to the publisher for the stream
in my_output_stream_id
'''
# Get the stream(s)
stream_id = self.CFG.get('process',{}).get('stream_id','')
self.greenlet_queue = []
self._usgs_def = USGS_stream_definition()
# Stream creation is done in SA, but to make the example go for demonstration create one here if it is not provided...
if not stream_id:
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
stream_id = pubsub_cli.create_stream(
name='Example USGS Data',
stream_definition=self._usgs_def,
original=True,
encoding='ION R2')
self.stream_publisher_registrar = StreamPublisherRegistrar(process=self,node=self.container.node)
# Needed to get the originator's stream_id
self.stream_id = stream_id
self.publisher = self.stream_publisher_registrar.create_publisher(stream_id=stream_id)
self.last_time = 0
g = Greenlet(self._trigger_func, stream_id)
log.warn('Starting publisher thread for simple usgs data.')
g.start()
self.greenlet_queue.append(g)
class PublishSubscribeIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
# Establish endpoint with container
container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
container_client.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to bank service
self.client = PubsubManagementServiceClient(node=self.container.node)
self.container.spawn_process('test_process', 'pyon.ion.streamproc','StreamProcess',
config={'process':{'type':'stream_process','listen_name':'ctd_data'}})
def test_create_publisher(self):
stream = IonObject(RT.Stream, name='test stream')
id = self.client.create_stream(stream)
def on_start(self):
log.warn('Entering On Start!!!')
# Get the stream(s)
stream_id = self.CFG.get_safe('process.stream_id',{})
self.greenlet_queue = []
# Stream creation is done in SA, but to make the example go for demonstration create one here if it is not provided...
if not stream_id:
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
stream_def_id = pubsub_cli.create_stream_definition(name='Producer stream %s' % str(uuid4()),container=self.outgoing_stream_def)
stream_id = pubsub_cli.create_stream(
name='Example CTD Data',
stream_definition_id = stream_def_id,
original=True,
encoding='ION R2')
self.stream_publisher_registrar = StreamPublisherRegistrar(process=self,node=self.container.node)
# Needed to get the originator's stream_id
self.stream_id= stream_id
self.publisher = self.stream_publisher_registrar.create_publisher(stream_id=stream_id)
self.last_time = 0
g = Greenlet(self._trigger_func, stream_id)
log.debug('Starting publisher thread for simple ctd data.')
g.start()
log.warn('Publisher Greenlet started in "%s"' % self.__class__.__name__)
self.greenlet_queue.append(g)
def on_start(self):
log.warn('Entering On Start!!!')
# Get the stream(s)
stream_id = self.CFG.get_safe('process.stream_id', {})
self.greenlet_queue = []
# Stream creation is done in SA, but to make the example go for demonstration create one here if it is not provided...
if not stream_id:
pubsub_cli = PubsubManagementServiceClient(
node=self.container.node)
stream_def_id = pubsub_cli.create_stream_definition(
name='Producer stream %s' % str(uuid4()),
container=self.outgoing_stream_def)
stream_id = pubsub_cli.create_stream(
name='Example CTD Data',
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self.stream_publisher_registrar = StreamPublisherRegistrar(
process=self, node=self.container.node)
# Needed to get the originator's stream_id
self.stream_id = stream_id
self.publisher = self.stream_publisher_registrar.create_publisher(
stream_id=stream_id)
self.last_time = 0
g = Greenlet(self._trigger_func, stream_id)
log.debug('Starting publisher thread for simple ctd data.')
g.start()
log.warn('Publisher Greenlet started in "%s"' %
self.__class__.__name__)
self.greenlet_queue.append(g)
def _build_stream_config(self):
"""
"""
if(not self.packet_config):
return
streams = self.packet_config
log.debug("Streams: %s", streams)
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
for stream_name in streams:
pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_PARAM_DICT, id_only=True)
if(not pd_id):
log.error("No pd_id found for param_dict '%s'" % DEFAULT_PARAM_DICT)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name,
parameter_dictionary_id=pd_id)
log.debug("Stream: %s (%s), stream_def_id %s" % (stream_name, type(stream_name), stream_def_id))
#pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
pd = None
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {
'parsed' : 'ctd_parsed_param_dict',
'raw' : 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data')
pd = get_param_dict(param_dict_name)
stream_config = dict(stream_route=stream_route,
stream_id=stream_id,
parameter_dictionary=pd.dump())
self._stream_config[stream_name] = stream_config
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
streams = {
'parsed': 'ctd_parsed_param_dict',
'raw': 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(
DEFAULT_PARAM_DICT, id_only=True)
if (not pd_id):
log.error("No pd_id found for param_dict '%s'" %
DEFAULT_PARAM_DICT)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
pd = None
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
def instrument_test_driver(container):
sa_user_header = container.governance_controller.get_system_actor_header()
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'mi.instrument.seabird.sbe37smb.ooicore.driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
_container_client = ContainerAgentClient(node=container.node,
name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, *args, **kwargs):
log.info('Subscriber received message: %s', str(message))
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
#@TODO: Figure out what goes inside this stream def, do we have a pdict?
stream_def_id = _pubsub_client.create_stream_definition(
name='instrument stream def')
stream_id, route = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
exchange_point='science_data')
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = StandaloneStreamSubscriber(exchange_name=exchange_name, callback=consume)
sub.start()
sub_id = _pubsub_client.create_subscription(\
name=exchange_name,
stream_ids=[stream_id])
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = '123xyz'
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': agent_resource_id}
}
# Launch an instrument agent process.
_ia_name = 'agent007'
_ia_mod = 'ion.agents.instrument.instrument_agent'
_ia_class = 'InstrumentAgent'
_ia_pid = _container_client.spawn_process(name=_ia_name,
module=_ia_mod, cls=_ia_class,
config=agent_config)
log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
class LastUpdateCacheTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.datastore_name = CACHE_DATASTORE_NAME
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA)
self.tms_cli = TransformManagementServiceClient()
self.pubsub_cli = PubsubManagementServiceClient()
self.pd_cli = ProcessDispatcherServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([bootstrap.get_sys_name(), xp_base])
except ValueError:
raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
def make_points(self,definition,stream_id='I am very special', N=100):
from prototype.sci_data.constructor_apis import PointSupplementConstructor
import numpy as np
import random
definition.stream_resource_id = stream_id
total = N
n = 10 # at most n records per granule
i = 0
while i < total:
r = random.randint(1,n)
psc = PointSupplementConstructor(point_definition=definition, stream_id=stream_id)
for x in xrange(r):
i+=1
point_id = psc.add_point(time=i, location=(0,0,0))
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='temperature', value=np.random.normal(loc=48.0,scale=4.0, size=1)[0])
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='pressure', value=np.float32(1.0))
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='conductivity', value=np.float32(2.0))
granule = psc.close_stream_granule()
hdf_string = granule.identifiables[definition.data_stream_id].values
granule.identifiables[definition.data_stream_id].values = hdf_string
yield granule
return
def start_worker(self):
proc_def = ProcessDefinition()
proc_def.executable['module'] = 'ion.processes.data.last_update_cache'
proc_def.executable['class'] = 'LastUpdateCache'
proc_def_id = self.pd_cli.create_process_definition(process_definition=proc_def)
subscription_id = self.pubsub_cli.create_subscription(query=ExchangeQuery(), exchange_name='ingestion_aggregate')
config = {
'couch_storage' : {
'datastore_name' :self.datastore_name,
'datastore_profile' : 'SCIDATA'
}
}
transform_id = self.tms_cli.create_transform(
name='last_update_cache',
description='LastUpdate that compiles an aggregate of metadata',
in_subscription_id=subscription_id,
process_definition_id=proc_def_id,
configuration=config
)
self.tms_cli.activate_transform(transform_id=transform_id)
transform = self.rr_cli.read(transform_id)
pid = transform.process_id
handle = self.container.proc_manager.procs[pid]
return handle
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_last_update_cache(self):
handle = self.start_worker()
queue = Queue()
o_process = handle.process
def new_process(msg):
o_process(msg)
queue.put(True)
handle.process = new_process
definition = SBE37_CDM_stream_definition()
publisher = Publisher()
stream_def_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id)
time = float(0.0)
for granule in self.make_points(definition=definition, stream_id=stream_id, N=10):
publisher.publish(granule, to_name=(self.XP, stream_id+'.data'))
# Determinism sucks
try:
queue.get(timeout=5)
except Empty:
self.assertTrue(False, 'Process never received the message.')
doc = self.db.read(stream_id)
ntime = doc.variables['time'].value
self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.')
time = ntime
class TransformPrototypeIntTest(IonIntegrationTestCase):
def setUp(self):
super(TransformPrototypeIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrc = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.ssclient = SchedulerServiceClient()
self.event_publisher = EventPublisher()
self.user_notification = UserNotificationServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.exchange_names = []
self.exchange_points = []
def tearDown(self):
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def now_utc(self):
return time.time()
def _create_interval_timer_with_end_time(self,timer_interval= None, end_time = None ):
'''
A convenience method to set up an interval timer with an end time
'''
self.timer_received_time = 0
self.timer_interval = timer_interval
start_time = self.now_utc()
if not end_time:
end_time = start_time + 2 * timer_interval + 1
log.debug("got the end time here!! %s" % end_time)
# Set up the interval timer. The scheduler will publish event with origin set as "Interval Timer"
sid = self.ssclient.create_interval_timer(start_time="now" ,
interval=self.timer_interval,
end_time=end_time,
event_origin="Interval Timer",
event_subtype="")
def cleanup_timer(scheduler, schedule_id):
"""
Do a friendly cancel of the scheduled event.
If it fails, it's ok.
"""
try:
scheduler.cancel_timer(schedule_id)
except:
log.warn("Couldn't cancel")
self.addCleanup(cleanup_timer, self.ssclient, sid)
return sid
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_event_processing(self):
'''
Test that events are processed by the transforms according to a provided algorithm
'''
#-------------------------------------------------------------------------------------
# Set up the scheduler for an interval timer with an end time
#-------------------------------------------------------------------------------------
id = self._create_interval_timer_with_end_time(timer_interval=2)
self.assertIsNotNone(id)
#-------------------------------------------------------------------------------------
# Create an event alert transform....
# The configuration for the Event Alert Transform... set up the event types to listen to
#-------------------------------------------------------------------------------------
configuration = {
'process':{
'event_type': 'ResourceEvent',
'timer_origin': 'Interval Timer',
'instrument_origin': 'My_favorite_instrument'
}
}
#-------------------------------------------------------------------------------------
# Create the process
#-------------------------------------------------------------------------------------
pid = TransformPrototypeIntTest.create_process( name= 'event_alert_transform',
module='ion.processes.data.transforms.event_alert_transform',
class_name='EventAlertTransform',
configuration= configuration)
self.addCleanup(self.process_dispatcher.cancel_process, pid)
self.assertIsNotNone(pid)
#-------------------------------------------------------------------------------------
# Publish events and make assertions about alerts
#-------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def event_received(message, headers):
queue.put(message)
event_subscriber = EventSubscriber( origin="EventAlertTransform",
event_type="DeviceEvent",
callback=event_received)
event_subscriber.start()
self.addCleanup(event_subscriber.stop)
# publish event twice
for i in xrange(5):
self.event_publisher.publish_event( event_type = 'ExampleDetectableEvent',
origin = "My_favorite_instrument",
voltage = 5,
telemetry = 10,
temperature = 20)
gevent.sleep(0.1)
self.assertTrue(queue.empty())
#publish event the third time but after a time interval larger than 2 seconds
gevent.sleep(5)
#-------------------------------------------------------------------------------------
# Make assertions about the alert event published by the EventAlertTransform
#-------------------------------------------------------------------------------------
event = queue.get(timeout=10)
log.debug("Alarm event received from the EventAertTransform %s" % event)
self.assertEquals(event.type_, "DeviceEvent")
self.assertEquals(event.origin, "EventAlertTransform")
#------------------------------------------------------------------------------------------------
# Now clear the event queue being populated by alarm events and publish normally once again
#------------------------------------------------------------------------------------------------
queue.queue.clear()
for i in xrange(5):
self.event_publisher.publish_event( event_type = 'ExampleDetectableEvent',
origin = "My_favorite_instrument",
voltage = 5,
telemetry = 10,
temperature = 20)
gevent.sleep(0.1)
self.assertTrue(queue.empty())
log.debug("This completes the requirement that the EventAlertTransform publishes \
an alarm event when it does not hear from the instrument for some time.")
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_stream_processing(self):
#--------------------------------------------------------------------------------
#Test that streams are processed by the transforms according to a provided algorithm
#--------------------------------------------------------------------------------
#todo: In this simple implementation, we are checking if the stream has the word, PUBLISH,
#todo(contd) and if the word VALUE=<number> exists and that number is less than something
#todo later on we are going to use complex algorithms to make this prototype powerful
#-------------------------------------------------------------------------------------
# Start a subscriber to listen for an alert event from the Stream Alert Transform
#-------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def event_received(message, headers):
queue.put(message)
event_subscriber = EventSubscriber( origin="StreamAlertTransform",
event_type="DeviceEvent",
callback=event_received)
event_subscriber.start()
self.addCleanup(event_subscriber.stop)
#-------------------------------------------------------------------------------------
# The configuration for the Stream Alert Transform... set up the event types to listen to
#-------------------------------------------------------------------------------------
config = {
'process':{
'queue_name': 'a_queue',
'value': 10,
'event_type':'DeviceEvent'
}
}
#-------------------------------------------------------------------------------------
# Create the process
#-------------------------------------------------------------------------------------
pid = TransformPrototypeIntTest.create_process( name= 'transform_data_process',
module='ion.processes.data.transforms.event_alert_transform',
class_name='StreamAlertTransform',
configuration= config)
self.addCleanup(self.process_dispatcher.cancel_process, pid)
self.assertIsNotNone(pid)
#-------------------------------------------------------------------------------------
# Publish streams and make assertions about alerts
#-------------------------------------------------------------------------------------
exchange_name = 'a_queue'
exchange_point = 'test_exchange'
routing_key = 'stream_id.stream'
stream_route = StreamRoute(exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(exchange_name)
xp = self.container.ex_manager.create_xp(exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
message = "A dummy example message containing the word PUBLISH, and with VALUE = 5 . This message" +\
" will trigger an alert event from the StreamAlertTransform because the value provided is "\
"less than 10 that was passed in through the config."
pub.publish(message)
event = queue.get(timeout=10)
self.assertEquals(event.type_, "DeviceEvent")
self.assertEquals(event.origin, "StreamAlertTransform")
# self.purge_queues(exchange_name)
# def purge_queues(self, exchange_name):
# xn = self.container.ex_manager.create_xn_queue(exchange_name)
# xn.purge()
@staticmethod
def create_process(name= '', module = '', class_name = '', configuration = None):
'''
A helper method to create a process
'''
producer_definition = ProcessDefinition(name=name)
producer_definition.executable = {
'module':module,
'class': class_name
}
process_dispatcher = ProcessDispatcherServiceClient()
procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
pid = process_dispatcher.schedule_process(process_definition_id= procdef_id, configuration=configuration)
return pid
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_demo_stream_granules_processing(self):
"""
Test that the Demo Stream Alert Transform is functioning. The transform coordinates with the scheduler.
It is configured to listen to a source that publishes granules. It publishes a DeviceStatusEvent if it
receives a granule with bad data or a DeviceCommsEvent if no granule has arrived between two timer events.
The transform is configured at launch using a config dictionary.
"""
#-------------------------------------------------------------------------------------
# Start a subscriber to listen for an alert event from the Stream Alert Transform
#-------------------------------------------------------------------------------------
queue_bad_data = gevent.queue.Queue()
queue_no_data = gevent.queue.Queue()
def bad_data(message, headers):
log.debug("Got a BAD data event: %s" % message)
if message.type_ == "DeviceStatusEvent":
queue_bad_data.put(message)
def no_data(message, headers):
log.debug("Got a NO data event: %s" % message)
queue_no_data.put(message)
event_subscriber_bad_data = EventSubscriber( origin="instrument_1",
event_type="DeviceStatusEvent",
callback=bad_data)
event_subscriber_no_data = EventSubscriber( origin="instrument_1",
event_type="DeviceCommsEvent",
callback=no_data)
event_subscriber_bad_data.start()
event_subscriber_no_data.start()
self.addCleanup(event_subscriber_bad_data.stop)
self.addCleanup(event_subscriber_no_data.stop)
#-------------------------------------------------------------------------------------
# The configuration for the Stream Alert Transform... set up the event types to listen to
#-------------------------------------------------------------------------------------
self.valid_values = [-100, 100]
self.timer_interval = 5
self.queue_name = 'a_queue'
config = {
'process':{
'timer_interval': self.timer_interval,
'queue_name': self.queue_name,
'variable_name': 'input_voltage',
'time_field_name': 'preferred_timestamp',
'valid_values': self.valid_values,
'timer_origin': 'Interval Timer',
'event_origin': 'instrument_1'
}
}
#-------------------------------------------------------------------------------------
# Create the process
#-------------------------------------------------------------------------------------
pid = TransformPrototypeIntTest.create_process( name= 'DemoStreamAlertTransform',
module='ion.processes.data.transforms.event_alert_transform',
class_name='DemoStreamAlertTransform',
configuration= config)
self.addCleanup(self.process_dispatcher.cancel_process, pid)
self.assertIsNotNone(pid)
#-------------------------------------------------------------------------------------
# Publish streams and make assertions about alerts
#-------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(name= 'platform_eng_parsed', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('demo_stream', parameter_dictionary_id=pdict_id)
stream_id, stream_route = self.pubsub_management.create_stream( name='test_demo_alert',
exchange_point='exch_point_1',
stream_definition_id=stream_def_id)
sub_1 = self.pubsub_management.create_subscription(name='sub_1', stream_ids=[stream_id], exchange_points=['exch_point_1'], exchange_name=self.queue_name)
self.pubsub_management.activate_subscription(sub_1)
self.exchange_names.append('sub_1')
self.exchange_points.append('exch_point_1')
#-------------------------------------------------------------------------------------
# publish a *GOOD* granule
#-------------------------------------------------------------------------------------
self.length = 2
val = numpy.array([random.uniform(0,50) for l in xrange(self.length)])
self._publish_granules(stream_id= stream_id, stream_route= stream_route, number=1, values=val)
self.assertTrue(queue_bad_data.empty())
#-------------------------------------------------------------------------------------
# publish a few *BAD* granules
#-------------------------------------------------------------------------------------
self.number = 2
val = numpy.array([(110 + l) for l in xrange(self.length)])
self._publish_granules(stream_id= stream_id, stream_route= stream_route, number= self.number, values=val)
for number in xrange(self.number):
event = queue_bad_data.get(timeout=40)
self.assertEquals(event.type_, "DeviceStatusEvent")
self.assertEquals(event.origin, "instrument_1")
self.assertEquals(event.state, DeviceStatusType.STATUS_WARNING)
self.assertEquals(event.valid_values, self.valid_values)
self.assertEquals(event.sub_type, 'input_voltage')
self.assertTrue(set(event.values) == set(val))
s = set(event.time_stamps)
cond = s in [set(numpy.array([1 for l in xrange(self.length)]).tolist()), set(numpy.array([2 for l in xrange(self.length)]).tolist())]
self.assertTrue(cond)
# To ensure that only the bad values generated the alert events. Queue should be empty now
self.assertEquals(queue_bad_data.qsize(), 0)
#-------------------------------------------------------------------------------------
# Do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status
#-------------------------------------------------------------------------------------
event = queue_no_data.get(timeout=15)
self.assertEquals(event.type_, "DeviceCommsEvent")
self.assertEquals(event.origin, "instrument_1")
self.assertEquals(event.origin_type, "PlatformDevice")
self.assertEquals(event.state, DeviceCommsType.DATA_DELIVERY_INTERRUPTION)
self.assertEquals(event.sub_type, 'input_voltage')
#-------------------------------------------------------------------------------------
# Empty the queues and repeat tests
#-------------------------------------------------------------------------------------
queue_bad_data.queue.clear()
queue_no_data.queue.clear()
#-------------------------------------------------------------------------------------
# publish a *GOOD* granule again
#-------------------------------------------------------------------------------------
val = numpy.array([(l + 20) for l in xrange(self.length)])
self._publish_granules(stream_id= stream_id, stream_route= stream_route, number=1, values=val)
self.assertTrue(queue_bad_data.empty())
#-------------------------------------------------------------------------------------
# Again do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status
#-------------------------------------------------------------------------------------
event = queue_no_data.get(timeout=20)
self.assertEquals(event.type_, "DeviceCommsEvent")
self.assertEquals(event.origin, "instrument_1")
self.assertEquals(event.origin_type, "PlatformDevice")
self.assertEquals(event.state, DeviceCommsType.DATA_DELIVERY_INTERRUPTION)
self.assertEquals(event.sub_type, 'input_voltage')
#-------------------------------------------------------------------------------------
# Again do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status
#-------------------------------------------------------------------------------------
ar = gevent.event.AsyncResult()
def poller(ar, method, *args):
events_in_db = method(*args)
if len(events_in_db) > 0:
ar.set(events_in_db)
return True
else:
return False
poll(poller, ar, self.user_notification.find_events, 'instrument_1')
# events_in_db = self.user_notification.find_events(origin='instrument_1')
events_in_db = ar.get(10)
log.debug("events::: %s" % events_in_db)
bad_data_events = []
no_data_events = []
for event in events_in_db:
if event.type_ == 'DeviceStatusEvent':
bad_data_events.append(event)
self.assertEquals(event.origin, "instrument_1")
self.assertEquals(event.status, DeviceStatusType.STATUS_WARNING)
self.assertEquals(event.valid_values, self.valid_values)
self.assertEquals(event.sub_type, 'input_voltage')
elif event.type_ == 'DeviceCommsEvent':
no_data_events.append(event)
self.assertEquals(event.origin, "instrument_1")
self.assertEquals(event.origin_type, "PlatformDevice")
self.assertEquals(event.status, DeviceCommsType.DATA_DELIVERY_INTERRUPTION)
self.assertEquals(event.sub_type, 'input_voltage')
self.assertTrue(len(bad_data_events) > 0)
self.assertTrue(len(no_data_events) > 0)
log.debug("This satisfies L4-CI-SA-RQ-114 : 'Marine facility shall monitor marine infrastructure usage by instruments.'"
" The req is satisfied because the stream alert transform"
"is able to send device status and communication events over selected time intervals. This capability will be "
"augmented in the future.")
def _publish_granules(self, stream_id=None, stream_route=None, values = None,number=None):
pub = StandaloneStreamPublisher(stream_id, stream_route)
stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id)
stream_def_id = stream_def._id
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
times = numpy.array([number for l in xrange(self.length)])
for i in xrange(number):
rdt['input_voltage'] = values
rdt['preferred_timestamp'] = ['time' for l in xrange(len(times))]
rdt['time'] = times
g = rdt.to_granule()
g.data_producer_id = 'instrument_1'
log.debug("granule #%s published by instrument:: %s" % ( number,g))
pub.publish(g)
@staticmethod
def makeEpochTime(date_time = None):
"""
provides the seconds since epoch give a python datetime object.
@param date_time Python datetime object
@retval seconds_since_epoch int
"""
seconds_since_epoch = calendar.timegm(date_time.timetuple())
return seconds_since_epoch
class EventTriggeredTransformIntTest(IonIntegrationTestCase):
def setUp(self):
super(EventTriggeredTransformIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self.queue_cleanup = []
self.exchange_cleanup = []
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.exchange_name = "test_queue"
self.exchange_point = "test_exchange"
self.dataset_management = DatasetManagementServiceClient()
def tearDown(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
for exchange in self.exchange_cleanup:
xp = self.container.ex_manager.create_xp(exchange)
xp.delete()
@attr("LOCOINT")
@unittest.skipIf(os.getenv("CEI_LAUNCH_TEST", False), "Skip test while in CEI LAUNCH mode")
def test_event_triggered_transform_A(self):
"""
Test that packets are processed by the event triggered transform
"""
# ---------------------------------------------------------------------------------------------
# Launch a ctd transform
# ---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name="EventTriggeredTransform_A", description="For testing EventTriggeredTransform_A"
)
process_definition.executable["module"] = "ion.processes.data.transforms.event_triggered_transform"
process_definition.executable["class"] = "EventTriggeredTransform_A"
event_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition
)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
stream_def_id = self.pubsub.create_stream_definition("cond_stream_def", parameter_dictionary_id=pdict_id)
cond_stream_id, _ = self.pubsub.create_stream(
"test_conductivity", exchange_point="science_data", stream_definition_id=stream_def_id
)
config.process.publish_streams.conductivity = cond_stream_id
config.process.event_type = "ResourceLifecycleEvent"
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=event_transform_proc_def_id, configuration=config
)
# ---------------------------------------------------------------------------------------------
# Publish an event to wake up the event triggered transform
# ---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ResourceLifecycleEvent")
event_publisher.publish_event(origin="fake_origin")
# ---------------------------------------------------------------------------------------------
# Create subscribers that will receive the conductivity, temperature and pressure granules from
# the ctd transform
# ---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber1(m, r, s):
ar_cond.set(m)
sub_event_transform = StandaloneStreamSubscriber("sub_event_transform", subscriber1)
self.addCleanup(sub_event_transform.stop)
sub_event_transform_id = self.pubsub.create_subscription(
"subscription_cond", stream_ids=[cond_stream_id], exchange_name="sub_event_transform"
)
self.pubsub.activate_subscription(sub_event_transform_id)
self.queue_cleanup.append(sub_event_transform.xn.queue)
sub_event_transform.start()
# ------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
# ------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = "stream_id.stream"
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind("stream_id.stream", xp)
pub = StandaloneStreamPublisher("stream_id", stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
# ------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
# ------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result_cond = ar_cond.get(timeout=10)
self.assertTrue(isinstance(result_cond, Granule))
rdt = RecordDictionaryTool.load_from_granule(result_cond)
self.assertTrue(rdt.__contains__("conductivity"))
self.check_cond_algorithm_execution(publish_granule, result_cond)
def check_cond_algorithm_execution(self, publish_granule, granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
output_data = output_rdt_transform["conductivity"]
input_data = input_rdt_to_transform["conductivity"]
self.assertTrue(((input_data / 100000.0) - 0.5).all() == output_data.all())
def _get_new_ctd_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
for field in rdt:
rdt[field] = numpy.array([random.uniform(0.0, 75.0) for i in xrange(length)])
g = rdt.to_granule()
return g
def test_event_triggered_transform_B(self):
"""
Test that packets are processed by the event triggered transform
"""
# ---------------------------------------------------------------------------------------------
# Launch a ctd transform
# ---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name="EventTriggeredTransform_B", description="For testing EventTriggeredTransform_B"
)
process_definition.executable["module"] = "ion.processes.data.transforms.event_triggered_transform"
process_definition.executable["class"] = "EventTriggeredTransform_B"
event_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition
)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
stream_def_id = self.pubsub.create_stream_definition("stream_def", parameter_dictionary_id=pdict_id)
stream_id, _ = self.pubsub.create_stream(
"test_stream", exchange_point="science_data", stream_definition_id=stream_def_id
)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
config.process.publish_streams.output = stream_id
config.process.event_type = "ResourceLifecycleEvent"
config.process.stream_id = stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=event_transform_proc_def_id, configuration=config
)
# ---------------------------------------------------------------------------------------------
# Publish an event to wake up the event triggered transform
# ---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ResourceLifecycleEvent")
event_publisher.publish_event(origin="fake_origin")
def test_usgs_integration(self):
'''
test_usgs_integration
Test full DM Services Integration using usgs
'''
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here
#-----------------------------
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(node=cc.node)
dataset_management_service = DatasetManagementServiceClient(node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
transform_management_service = TransformManagementServiceClient(node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
process_list = []
datasets = []
datastore_name = 'test_usgs_integration'
#---------------------------
# Set up ingestion
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,datastore_profile='SCIDATA'),
number_of_workers=8
)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
usgs_stream_def = USGS_stream_definition()
stream_def_id = pubsub_management_service.create_stream_definition(container=usgs_stream_def, name='Junk definition')
#---------------------------
# Set up the producers (CTD Simulators)
#---------------------------
# Launch five simulated CTD producers
for iteration in xrange(2):
# Make a stream to output on
stream_id = pubsub_management_service.create_stream(stream_definition_id=stream_def_id)
#---------------------------
# Set up the datasets
#---------------------------
dataset_id = dataset_management_service.create_dataset(
stream_id=stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule'
)
# Keep track of the datasets
datasets.append(dataset_id)
stream_policy_id = ingestion_management_service.create_dataset_configuration(
dataset_id = dataset_id,
archive_data = True,
archive_metadata = True,
ingestion_configuration_id = ingestion_configuration_id
)
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':'ion.agents.eoi.handler.usgs_stream_publisher',
'class':'UsgsPublisher'
}
configuration = {
'process':{
'stream_id':stream_id,
}
}
procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
log.debug('LUKE_DEBUG: procdef_id: %s', procdef_id)
pid = process_dispatcher.schedule_process(process_definition_id=procdef_id, configuration=configuration)
# Keep track, we'll kill 'em later.
process_list.append(pid)
# Get about 4 seconds of data
time.sleep(4)
#---------------------------
# Stop producing data
#---------------------------
for process in process_list:
process_dispatcher.cancel_process(process)
#----------------------------------------------
# The replay and the transform, a love story.
#----------------------------------------------
# Happy Valentines to the clever coder who catches the above!
transform_definition = ProcessDefinition()
transform_definition.executable = {
'module':'ion.processes.data.transforms.transform_example',
'class':'TransformCapture'
}
transform_definition_id = process_dispatcher.create_process_definition(process_definition=transform_definition)
dataset_id = datasets.pop() # Just need one for now
replay_id, stream_id = data_retriever_service.define_replay(dataset_id=dataset_id)
#--------------------------------------------
# I'm Selling magazine subscriptions here!
#--------------------------------------------
subscription = pubsub_management_service.create_subscription(query=StreamQuery(stream_ids=[stream_id]),
exchange_name='transform_capture_point')
#--------------------------------------------
# Start the transform (capture)
#--------------------------------------------
transform_id = transform_management_service.create_transform(
name='capture_transform',
in_subscription_id=subscription,
process_definition_id=transform_definition_id
)
transform_management_service.activate_transform(transform_id=transform_id)
#--------------------------------------------
# BEGIN REPLAY!
#--------------------------------------------
data_retriever_service.start_replay(replay_id=replay_id)
#--------------------------------------------
# Lets get some boundaries
#--------------------------------------------
bounds = dataset_management_service.get_dataset_bounds(dataset_id=dataset_id)
class TestTransformPrime(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url(
'res/deploy/r2deploy.yml') # Because hey why not?!
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
def setup_streams(self):
in_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'sbe37_L0_test', id_only=True)
out_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'sbe37_L1_test', id_only=True)
in_stream_def_id = self.pubsub_management.create_stream_definition(
'L0 SBE37', parameter_dictionary_id=in_pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
in_stream_def_id)
out_stream_def_id = self.pubsub_management.create_stream_definition(
'L1 SBE37', parameter_dictionary_id=out_pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
out_stream_def_id)
in_stream_id, in_route = self.pubsub_management.create_stream(
'L0 input',
stream_definition_id=in_stream_def_id,
exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, in_stream_id)
out_stream_id, out_route = self.pubsub_management.create_stream(
'L0 output',
stream_definition_id=out_stream_def_id,
exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, out_stream_id)
return [(in_stream_id, in_stream_def_id),
(out_stream_id, out_stream_def_id)]
def setup_advanced_streams(self):
in_pdict_id = out_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'sbe37_LC_TEST', id_only=True)
in_stream_def_id = self.pubsub_management.create_stream_definition(
'sbe37_instrument',
parameter_dictionary_id=in_pdict_id,
available_fields=[
'time', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0', 'lat', 'lon'
])
self.addCleanup(self.pubsub_management.delete_stream_definition,
in_stream_def_id)
out_stream_def_id = self.pubsub_management.create_stream_definition(
'sbe37_l2',
parameter_dictionary_id=out_pdict_id,
available_fields=['time', 'rho', 'PRACSAL_L2'])
self.addCleanup(self.pubsub_management.delete_stream_definition,
out_stream_def_id)
in_stream_id, in_route = self.pubsub_management.create_stream(
'instrument stream',
stream_definition_id=in_stream_def_id,
exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, in_stream_id)
out_stream_id, out_route = self.pubsub_management.create_stream(
'data product stream',
stream_definition_id=out_stream_def_id,
exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, out_stream_id)
return [(in_stream_id, in_stream_def_id),
(out_stream_id, out_stream_def_id)]
def preload(self):
config = DotDict()
config.op = 'load'
config.scenario = 'BASE,LC_TEST'
config.categories = 'ParameterFunctions,ParameterDefs,ParameterDictionary'
config.path = 'res/preload/r2_ioc'
self.container.spawn_process('preload',
'ion.processes.bootstrap.ion_loader',
'IONLoader', config)
def setup_advanced_transform(self):
self.preload()
queue_name = 'transform_prime'
stream_info = self.setup_advanced_streams()
in_stream_id, in_stream_def_id = stream_info[0]
out_stream_id, out_stream_def_id = stream_info[1]
routes = {}
routes[(in_stream_id, out_stream_id)] = None
config = DotDict()
config.process.queue_name = queue_name
config.process.routes = routes
config.process.publish_streams = {out_stream_id: out_stream_id}
sub_id = self.pubsub_management.create_subscription(
queue_name, stream_ids=[in_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
self.container.spawn_process(
'transform_prime', 'ion.processes.data.transforms.transform_prime',
'TransformPrime', config)
listen_sub_id = self.pubsub_management.create_subscription(
'listener', stream_ids=[out_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription,
listen_sub_id)
self.pubsub_management.activate_subscription(listen_sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription,
listen_sub_id)
return [(in_stream_id, in_stream_def_id),
(out_stream_id, out_stream_def_id)]
def setup_transform(self):
self.preload()
queue_name = 'transform_prime'
stream_info = self.setup_streams()
in_stream_id, in_stream_def_id = stream_info[0]
out_stream_id, out_stream_def_id = stream_info[1]
routes = {}
routes[(in_stream_id, out_stream_id)] = None
config = DotDict()
config.process.queue_name = queue_name
config.process.routes = routes
config.process.publish_streams = {out_stream_id: out_stream_id}
sub_id = self.pubsub_management.create_subscription(
queue_name, stream_ids=[in_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
self.container.spawn_process(
'transform_prime', 'ion.processes.data.transforms.transform_prime',
'TransformPrime', config)
listen_sub_id = self.pubsub_management.create_subscription(
'listener', stream_ids=[out_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription,
listen_sub_id)
self.pubsub_management.activate_subscription(listen_sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription,
listen_sub_id)
return [(in_stream_id, in_stream_def_id),
(out_stream_id, out_stream_def_id)]
def setup_validator(self, validator):
listener = StandaloneStreamSubscriber('listener', validator)
listener.start()
self.addCleanup(listener.stop)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_execute_advanced_transform(self):
# Runs a transform across L0-L2 with stream definitions including available fields
streams = self.setup_advanced_transform()
in_stream_id, in_stream_def_id = streams[0]
out_stream_id, out_stream_defs_id = streams[1]
validation_event = Event()
def validator(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
if not np.allclose(rdt['rho'], np.array([1001.0055034])):
return
validation_event.set()
self.setup_validator(validator)
in_route = self.pubsub_management.read_stream_route(in_stream_id)
publisher = StandaloneStreamPublisher(in_stream_id, in_route)
outbound_rdt = RecordDictionaryTool(
stream_definition_id=in_stream_def_id)
outbound_rdt['time'] = [0]
outbound_rdt['TEMPWAT_L0'] = [280000]
outbound_rdt['CONDWAT_L0'] = [100000]
outbound_rdt['PRESWAT_L0'] = [2789]
outbound_rdt['lat'] = [45]
outbound_rdt['lon'] = [-71]
outbound_granule = outbound_rdt.to_granule()
publisher.publish(outbound_granule)
self.assertTrue(validation_event.wait(2))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_execute_transform(self):
streams = self.setup_transform()
in_stream_id, in_stream_def_id = streams[0]
out_stream_id, out_stream_def_id = streams[1]
validation_event = Event()
def validator(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
if not np.allclose(rdt['TEMPWAT_L1'], np.array([18.])):
return
if not np.allclose(rdt['CONDWAT_L1'], np.array([0.5])):
return
if not np.allclose(rdt['PRESWAT_L1'], np.array([0.04536611])):
return
validation_event.set()
self.setup_validator(validator)
in_route = self.pubsub_management.read_stream_route(in_stream_id)
publisher = StandaloneStreamPublisher(in_stream_id, in_route)
outbound_rdt = RecordDictionaryTool(
stream_definition_id=in_stream_def_id)
outbound_rdt['time'] = [0]
outbound_rdt['TEMPWAT_L0'] = [280000]
outbound_rdt['CONDWAT_L0'] = [100000]
outbound_rdt['PRESWAT_L0'] = [2789]
outbound_rdt['lat'] = [45]
outbound_rdt['lon'] = [-71]
outbound_granule = outbound_rdt.to_granule()
publisher.publish(outbound_granule)
self.assertTrue(validation_event.wait(2))
def test_dm_integration(self):
'''
test_salinity_transform
Test full DM Services Integration
'''
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here to run as a script (don't forget the imports of course!)
#-----------------------------
# Create some service clients...
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(
node=cc.node)
dataset_management_service = DatasetManagementServiceClient(
node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
transform_management_service = TransformManagementServiceClient(
node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
# declare some handy variables
datastore_name = 'test_dm_integration'
###
### In the beginning there were two stream definitions...
###
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = pubsub_management_service.create_stream_definition(
container=ctd_stream_def, name='Simulated CTD data')
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = pubsub_management_service.create_stream_definition(
container=SalinityTransform.outgoing_stream_def,
name='Scalar Salinity data stream')
###
### And two process definitions...
###
# one for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.ctd_stream_publisher',
'class': 'SimpleCtdPublisher'
}
ctd_sim_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
# one for the salinity transform
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.transforms.ctd.ctd_L2_salinity',
'class': 'SalinityTransform'
}
salinity_transform_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
#---------------------------
# Set up ingestion - this is an operator concern - not done by SA in a deployed system
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,
datastore_profile='SCIDATA'),
number_of_workers=1)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
#---------------------------
# Set up the producer (CTD Simulator)
#---------------------------
# Create the stream
ctd_stream_id = pubsub_management_service.create_stream(
stream_definition_id=ctd_stream_def_id)
# Set up the datasets
ctd_dataset_id = dataset_management_service.create_dataset(
stream_id=ctd_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of this dataset
ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=ctd_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
#---------------------------
# Set up the salinity transform
#---------------------------
# Create the stream
sal_stream_id = pubsub_management_service.create_stream(
stream_definition_id=sal_stream_def_id)
# Set up the datasets
sal_dataset_id = dataset_management_service.create_dataset(
stream_id=sal_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of the salinity as a dataset
sal_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=sal_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto sal_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
# Create a subscription as input to the transform
sal_transform_input_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery(stream_ids=[
ctd_stream_id,
]),
exchange_name='salinity_transform_input'
) # how do we make these names??? i.e. Should they be anonymous?
# create the salinity transform
sal_transform_id = transform_management_service.create_transform(
name='example salinity transform',
in_subscription_id=sal_transform_input_subscription_id,
out_streams={
'output': sal_stream_id,
},
process_definition_id=salinity_transform_procdef_id,
# no configuration needed at this time...
)
# start the transform - for a test case it makes sense to do it before starting the producer but it is not required
transform_management_service.activate_transform(
transform_id=sal_transform_id)
# Start the ctd simulator to produce some data
configuration = {
'process': {
'stream_id': ctd_stream_id,
}
}
ctd_sim_pid = process_dispatcher.schedule_process(
process_definition_id=ctd_sim_procdef_id,
configuration=configuration)
###
### Make a subscriber in the test to listen for salinity data
###
salinity_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery([
sal_stream_id,
]),
exchange_name='salinity_test',
name="test salinity subscription",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process,
node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn('Salinity data received!')
results.append(message)
if len(results) > 3:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(
exchange_name='salinity_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
pubsub_management_service.activate_subscription(
subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=10))
# stop the flow parse the messages...
process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
for message in results:
psd = PointSupplementStreamParser(
stream_definition=SalinityTransform.outgoing_stream_def,
stream_granule=message)
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
import numpy
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) >
0.0) # salinity should always be greater than 0
def run_basic_transform(self):
''' Runs a basic example of a transform. It chains two transforms together, each add 1 to their input
Producer -> A -> B
Producer generates a number every four seconds and publishes it on the 'ctd_output_stream'
the producer is acting as a CTD or instrument in this example.
A is a basic transform that increments its input and publishes it on the 'transform_output' stream.
B is a basic transform that receives input.
All transforms write logging data to 'FS.TEMP/transform_output' so you can visually see activity of the transforms
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = IonObject(RT.ProcessDefinition,
name='transform_process_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class': 'TransformExample'
}
process_definition_id = procd_cli.create_process_definition(
process_definition)
#-------------------------------
# First Transform
#-------------------------------
# Create a dummy output stream from a 'ctd' instrument
ctd_output_stream_id = pubsub_cli.create_stream(
name='ctd_output_stream', original=True)
# Create the subscription to the ctd_output_stream
query = StreamQuery(stream_ids=[ctd_output_stream_id])
ctd_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='ctd_output')
# Create an output stream for the transform
transform_output_stream_id = pubsub_cli.create_stream(
name='transform_output', original=True)
configuration = {}
# Launch the first transform process
transform_id = tms_cli.create_transform(
name='basic_transform',
in_subscription_id=ctd_subscription_id,
out_streams={'output': transform_output_stream_id},
process_definition_id=process_definition_id,
configuration=configuration)
tms_cli.activate_transform(transform_id)
#-------------------------------
# Second Transform
#-------------------------------
# Create a SUBSCRIPTION to this output stream for the second transform
query = StreamQuery(stream_ids=[transform_output_stream_id])
second_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='final_output')
# Create a final output stream
final_output_id = pubsub_cli.create_stream(name='final_output',
original=True)
configuration = {}
second_transform_id = tms_cli.create_transform(
name='second_transform',
in_subscription_id=second_subscription_id,
out_streams={'output': final_output_id},
process_definition_id=process_definition_id,
configuration=configuration)
tms_cli.activate_transform(second_transform_id)
#-------------------------------
# Producer (Sample Input)
#-------------------------------
# Create a producing example process
id_p = self.container.spawn_process(
'myproducer', 'ion.processes.data.transforms.transform_example',
'TransformExampleProducer', {
'process': {
'type': 'stream_process',
'publish_streams': {
'out_stream': ctd_output_stream_id
}
},
'stream_producer': {
'interval': 4000
}
})
self.container.proc_manager.procs[id_p].start()
class TransformPrototypeIntTest(IonIntegrationTestCase):
def setUp(self):
super(TransformPrototypeIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrc = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.ssclient = SchedulerServiceClient()
self.event_publisher = EventPublisher()
self.exchange_names = []
self.exchange_points = []
def tearDown(self):
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def now_utc(self):
return time.mktime(datetime.datetime.utcnow().timetuple())
def _create_interval_timer_with_end_time(self,timer_interval= None, end_time = None ):
'''
A convenience method to set up an interval timer with an end time
'''
self.timer_received_time = 0
self.timer_interval = timer_interval
start_time = self.now_utc()
if not end_time:
end_time = start_time + 2 * timer_interval + 1
log.debug("got the end time here!! %s" % end_time)
# Set up the interval timer. The scheduler will publish event with origin set as "Interval Timer"
sid = self.ssclient.create_interval_timer(start_time="now" ,
interval=self.timer_interval,
end_time=end_time,
event_origin="Interval Timer",
event_subtype="")
def cleanup_timer(scheduler, schedule_id):
"""
Do a friendly cancel of the scheduled event.
If it fails, it's ok.
"""
try:
scheduler.cancel_timer(schedule_id)
except:
log.warn("Couldn't cancel")
self.addCleanup(cleanup_timer, self.ssclient, sid)
return sid
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_event_processing(self):
'''
Test that events are processed by the transforms according to a provided algorithm
'''
#-------------------------------------------------------------------------------------
# Set up the scheduler for an interval timer with an end time
#-------------------------------------------------------------------------------------
id = self._create_interval_timer_with_end_time(timer_interval=2)
self.assertIsNotNone(id)
#-------------------------------------------------------------------------------------
# Create an event alert transform....
# The configuration for the Event Alert Transform... set up the event types to listen to
#-------------------------------------------------------------------------------------
configuration = {
'process':{
'event_type': 'ResourceEvent',
'timer_origin': 'Interval Timer',
'instrument_origin': 'My_favorite_instrument'
}
}
#-------------------------------------------------------------------------------------
# Create the process
#-------------------------------------------------------------------------------------
pid = TransformPrototypeIntTest.create_process( name= 'event_alert_transform',
module='ion.processes.data.transforms.event_alert_transform',
class_name='EventAlertTransform',
configuration= configuration)
self.assertIsNotNone(pid)
#-------------------------------------------------------------------------------------
# Publish events and make assertions about alerts
#-------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def event_received(message, headers):
queue.put(message)
event_subscriber = EventSubscriber( origin="EventAlertTransform",
event_type="DeviceEvent",
callback=event_received)
event_subscriber.start()
self.addCleanup(event_subscriber.stop)
# publish event twice
for i in xrange(5):
self.event_publisher.publish_event( event_type = 'ExampleDetectableEvent',
origin = "My_favorite_instrument",
voltage = 5,
telemetry = 10,
temperature = 20)
gevent.sleep(0.1)
self.assertTrue(queue.empty())
#publish event the third time but after a time interval larger than 2 seconds
gevent.sleep(5)
#-------------------------------------------------------------------------------------
# Make assertions about the alert event published by the EventAlertTransform
#-------------------------------------------------------------------------------------
event = queue.get(timeout=10)
log.debug("Alarm event received from the EventAertTransform %s" % event)
self.assertEquals(event.type_, "DeviceEvent")
self.assertEquals(event.origin, "EventAlertTransform")
#------------------------------------------------------------------------------------------------
# Now clear the event queue being populated by alarm events and publish normally once again
#------------------------------------------------------------------------------------------------
queue.queue.clear()
for i in xrange(5):
self.event_publisher.publish_event( event_type = 'ExampleDetectableEvent',
origin = "My_favorite_instrument",
voltage = 5,
telemetry = 10,
temperature = 20)
gevent.sleep(0.1)
self.assertTrue(queue.empty())
log.debug("This completes the requirement that the EventAlertTransform publishes \
an alarm event when it does not hear from the instrument for some time.")
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_stream_processing(self):
#--------------------------------------------------------------------------------
#Test that streams are processed by the transforms according to a provided algorithm
#--------------------------------------------------------------------------------
#todo: In this simple implementation, we are checking if the stream has the word, PUBLISH,
#todo(contd) and if the word VALUE=<number> exists and that number is less than something
#todo later on we are going to use complex algorithms to make this prototype powerful
#-------------------------------------------------------------------------------------
# Start a subscriber to listen for an alert event from the Stream Alert Transform
#-------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def event_received(message, headers):
queue.put(message)
event_subscriber = EventSubscriber( origin="StreamAlertTransform",
event_type="DeviceEvent",
callback=event_received)
event_subscriber.start()
self.addCleanup(event_subscriber.stop)
#-------------------------------------------------------------------------------------
# The configuration for the Stream Alert Transform... set up the event types to listen to
#-------------------------------------------------------------------------------------
config = {
'process':{
'queue_name': 'a_queue',
'value': 10,
'event_type':'DeviceEvent'
}
}
#-------------------------------------------------------------------------------------
# Create the process
#-------------------------------------------------------------------------------------
pid = TransformPrototypeIntTest.create_process( name= 'transform_data_process',
module='ion.processes.data.transforms.event_alert_transform',
class_name='StreamAlertTransform',
configuration= config)
self.assertIsNotNone(pid)
#-------------------------------------------------------------------------------------
# Publish streams and make assertions about alerts
#-------------------------------------------------------------------------------------
exchange_name = 'a_queue'
exchange_point = 'test_exchange'
routing_key = 'stream_id.stream'
stream_route = StreamRoute(exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(exchange_name)
xp = self.container.ex_manager.create_xp(exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
message = "A dummy example message containing the word PUBLISH, and with VALUE = 5 . This message" +\
" will trigger an alert event from the StreamAlertTransform because the value provided is "\
"less than 10 that was passed in through the config."
pub.publish(message)
event = queue.get(timeout=10)
self.assertEquals(event.type_, "DeviceEvent")
self.assertEquals(event.origin, "StreamAlertTransform")
# self.purge_queues(exchange_name)
# def purge_queues(self, exchange_name):
# xn = self.container.ex_manager.create_xn_queue(exchange_name)
# xn.purge()
@staticmethod
def create_process(name= '', module = '', class_name = '', configuration = None):
'''
A helper method to create a process
'''
producer_definition = ProcessDefinition(name=name)
producer_definition.executable = {
'module':module,
'class': class_name
}
process_dispatcher = ProcessDispatcherServiceClient()
procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
pid = process_dispatcher.schedule_process(process_definition_id= procdef_id, configuration=configuration)
return pid
def test_demo_stream_granules_processing(self):
"""
Test that the Demo Stream Alert Transform is functioning. The transform coordinates with the scheduler.
It is configured to listen to a source that publishes granules. It publishes a DeviceStatusEvent if it
receives a granule with bad data or a DeviceCommsEvent if no granule has arrived between two timer events.
The transform is configured at launch using a config dictionary.
"""
#-------------------------------------------------------------------------------------
# Start a subscriber to listen for an alert event from the Stream Alert Transform
#-------------------------------------------------------------------------------------
queue_bad_data = gevent.queue.Queue()
queue_no_data = gevent.queue.Queue()
def bad_data(message, headers):
if message.type_ == "DeviceStatusEvent":
queue_bad_data.put(message)
def no_data(message, headers):
queue_no_data.put(message)
event_subscriber_bad_data = EventSubscriber( origin="DemoStreamAlertTransform",
event_type="DeviceStatusEvent",
callback=bad_data)
event_subscriber_no_data = EventSubscriber( origin="DemoStreamAlertTransform",
event_type="DeviceCommsEvent",
callback=no_data)
event_subscriber_bad_data.start()
event_subscriber_no_data.start()
self.addCleanup(event_subscriber_bad_data.stop)
self.addCleanup(event_subscriber_no_data.stop)
#-------------------------------------------------------------------------------------
# The configuration for the Stream Alert Transform... set up the event types to listen to
#-------------------------------------------------------------------------------------
self.valid_values = [-100, 100]
self.timer_interval = 5
self.queue_name = 'a_queue'
config = {
'process':{
'timer_interval': self.timer_interval,
'queue_name': self.queue_name,
'variable_name': 'input_voltage',
'time_field_name': 'preferred_timestamp',
'valid_values': self.valid_values,
'timer_origin': 'Interval Timer'
}
}
#-------------------------------------------------------------------------------------
# Create the process
#-------------------------------------------------------------------------------------
pid = TransformPrototypeIntTest.create_process( name= 'DemoStreamAlertTransform',
module='ion.processes.data.transforms.event_alert_transform',
class_name='DemoStreamAlertTransform',
configuration= config)
self.assertIsNotNone(pid)
#-------------------------------------------------------------------------------------
# Publish streams and make assertions about alerts
#-------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(name= 'platform_eng_parsed', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('demo_stream', parameter_dictionary_id=pdict_id)
stream_id, stream_route = self.pubsub_management.create_stream( name='test_demo_alert',
exchange_point='exch_point_1',
stream_definition_id=stream_def_id)
sub_1 = self.pubsub_management.create_subscription(name='sub_1', stream_ids=[stream_id], exchange_points=['exch_point_1'], exchange_name=self.queue_name)
self.pubsub_management.activate_subscription(sub_1)
self.exchange_names.append('sub_1')
self.exchange_points.append('exch_point_1')
#-------------------------------------------------------------------------------------
# publish a *GOOD* granule
#-------------------------------------------------------------------------------------
self.length = 2
val = numpy.array([random.uniform(0,50) for l in xrange(self.length)])
self._publish_granules(stream_id= stream_id, stream_route= stream_route, number=1, values=val, length=self.length)
self.assertTrue(queue_bad_data.empty())
#-------------------------------------------------------------------------------------
# publish a few *BAD* granules
#-------------------------------------------------------------------------------------
self.length = 2
self.number = 2
val = numpy.array([random.uniform(110,200) for l in xrange(self.length)])
self._publish_granules(stream_id= stream_id, stream_route= stream_route, number= self.number, values=val, length=self.length)
for i in xrange(self.length * self.number):
event = queue_bad_data.get(timeout=10)
self.assertEquals(event.type_, "DeviceStatusEvent")
self.assertEquals(event.origin, "DemoStreamAlertTransform")
self.assertEquals(event.state, DeviceStatusType.OUT_OF_RANGE)
self.assertEquals(event.valid_values, self.valid_values)
self.assertEquals(event.sub_type, 'input_voltage')
self.assertIsNotNone(event.value)
self.assertIsNotNone(event.time_stamp)
# To ensure that only the bad values generated the alert events. Queue should be empty now
self.assertEquals(queue_bad_data.qsize(), 0)
#-------------------------------------------------------------------------------------
# Do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status
#-------------------------------------------------------------------------------------
event = queue_no_data.get(timeout=15)
self.assertEquals(event.type_, "DeviceCommsEvent")
self.assertEquals(event.origin, "DemoStreamAlertTransform")
self.assertEquals(event.state, DeviceCommsType.DATA_DELIVERY_INTERRUPTION)
self.assertEquals(event.sub_type, 'input_voltage')
#-------------------------------------------------------------------------------------
# Empty the queues and repeat tests
#-------------------------------------------------------------------------------------
queue_bad_data.queue.clear()
queue_no_data.queue.clear()
#-------------------------------------------------------------------------------------
# publish a *GOOD* granule again
#-------------------------------------------------------------------------------------
self.length = 2
val = numpy.array([random.uniform(0,50) for l in xrange(self.length)])
self._publish_granules(stream_id= stream_id, stream_route= stream_route, number=1, values=val, length=self.length)
self.assertTrue(queue_bad_data.empty())
#-------------------------------------------------------------------------------------
# Again do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status
#-------------------------------------------------------------------------------------
event = queue_no_data.get(timeout=20)
self.assertEquals(event.type_, "DeviceCommsEvent")
self.assertEquals(event.origin, "DemoStreamAlertTransform")
self.assertEquals(event.state, DeviceCommsType.DATA_DELIVERY_INTERRUPTION)
self.assertEquals(event.sub_type, 'input_voltage')
def _publish_granules(self, stream_id=None, stream_route=None, values = None,number=None, length=None):
pub = StandaloneStreamPublisher(stream_id, stream_route)
stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id)
stream_def_id = stream_def._id
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
for i in xrange(number):
rdt['input_voltage'] = values
rdt['preferred_timestamp'] = numpy.array([random.uniform(0,1000) for l in xrange(length)])
g = rdt.to_granule()
pub.publish(g)
class TransformManagementServiceIntTest(IonIntegrationTestCase):
def setUp(self):
# set up the container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.tms_cli = TransformManagementServiceClient(node=self.container.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.container.node)
self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.input_stream_id = self.pubsub_cli.create_stream(name='input_stream',original=True)
self.input_subscription_id = self.pubsub_cli.create_subscription(query=StreamQuery(stream_ids=[self.input_stream_id]),exchange_name='transform_input',name='input_subscription')
self.output_stream_id = self.pubsub_cli.create_stream(name='output_stream',original=True)
self.process_definition = ProcessDefinition(name='basic_transform_definition')
self.process_definition.executable = {'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'}
self.process_definition_id = self.procd_cli.create_process_definition(process_definition=self.process_definition)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_create_transform(self):
configuration = {'program_args':{'arg1':'value'}}
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id)
# test transform creation in rr
transform = self.rr_cli.read(transform_id)
self.assertEquals(transform.name,'test_transform')
# test associations
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),3)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc,TransformExample)
# clean up
self.tms_cli.delete_transform(transform_id)
def test_create_transform_no_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',in_subscription_id=self.input_subscription_id)
def test_create_transform_bad_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',
in_subscription_id=self.input_subscription_id,
process_definition_id='bad')
def test_create_transform_no_config(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
def test_create_transform_name_failure(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
with self.assertRaises(BadRequest):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_create_no_output(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id,
)
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),2)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc,TransformExample)
self.tms_cli.delete_transform(transform_id)
def test_read_transform_exists(self):
trans_obj = IonObject(RT.Transform,name='trans_obj')
trans_id, _ = self.rr_cli.create(trans_obj)
res = self.tms_cli.read_transform(trans_id)
actual = self.rr_cli.read(trans_id)
self.assertEquals(res._id,actual._id)
def test_read_transform_nonexist(self):
with self.assertRaises(NotFound) as e:
self.tms_cli.read_transform('123')
def test_activate_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id
)
self.tms_cli.activate_transform(transform_id)
# pubsub check if activated?
self.tms_cli.delete_transform(transform_id)
def test_activate_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.activate_transform('1234')
def test_delete_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id
)
self.tms_cli.delete_transform(transform_id)
# assertions
with self.assertRaises(NotFound):
self.rr_cli.read(transform_id)
def test_delete_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.delete_transform('123')
def test_execute_transform(self):
# set up
process_definition = ProcessDefinition(name='procdef_execute')
process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ReverseTransform'
data = [1,2,3]
process_definition_id, _ = self.rr_cli.create(process_definition)
retval = self.tms_cli.execute_transform(process_definition_id,data)
self.assertEquals(retval,[3,2,1])
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
"""
class RecordDictionaryIntegrationTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management = DatasetManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.rdt = None
self.data_producer_id = None
self.provider_metadata_update = None
self.event = Event()
def verify_incoming(self, m,r,s):
rdt = RecordDictionaryTool.load_from_granule(m)
for k,v in rdt.iteritems():
np.testing.assert_array_equal(v, self.rdt[k])
self.assertEquals(m.data_producer_id, self.data_producer_id)
self.assertEquals(m.provider_metadata_update, self.provider_metadata_update)
self.assertNotEqual(m.creation_timestamp, None)
self.event.set()
def test_serialize_compatability(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('ctd extended', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('ctd1', 'xp1', stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
sub_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
verified = Event()
def verifier(msg, route, stream_id):
for k,v in msg.record_dictionary.iteritems():
if v is not None:
self.assertIsInstance(v, np.ndarray)
rdt = RecordDictionaryTool.load_from_granule(msg)
for k,v in rdt.iteritems():
self.assertIsInstance(rdt[k], np.ndarray)
self.assertIsInstance(v, np.ndarray)
verified.set()
subscriber = StandaloneStreamSubscriber('sub1', callback=verifier)
subscriber.start()
self.addCleanup(subscriber.stop)
publisher = StandaloneStreamPublisher(stream_id,route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
ph.fill_rdt(rdt,10)
publisher.publish(rdt.to_granule())
self.assertTrue(verified.wait(60))
def test_granule(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':"GA03FLMA-RI001-13-CTDMOG999"})
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
self.addCleanup(self.pubsub_management.delete_stream_definition,stream_def_id)
stream_id, route = self.pubsub_management.create_stream('ctd_stream', 'xp1', stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream,stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
subscriber = StandaloneStreamSubscriber('sub', self.verify_incoming)
subscriber.start()
self.addCleanup(subscriber.stop)
subscription_id = self.pubsub_management.create_subscription('sub', stream_ids=[stream_id])
self.pubsub_management.activate_subscription(subscription_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.random.randn(10) * 10 + 30
rdt['pressure'] = [20] * 10
self.assertEquals(set(pdict.keys()), set(rdt.fields))
self.assertEquals(pdict.temporal_parameter_name, rdt.temporal_parameter)
self.assertEquals(rdt._stream_config['reference_designator'],"GA03FLMA-RI001-13-CTDMOG999")
self.rdt = rdt
self.data_producer_id = 'data_producer'
self.provider_metadata_update = {1:1}
publisher.publish(rdt.to_granule(data_producer_id='data_producer', provider_metadata_update={1:1}))
self.assertTrue(self.event.wait(10))
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.array([None,None,None])
self.assertTrue(rdt['time'] is None)
rdt['time'] = np.array([None, 1, 2])
self.assertEquals(rdt['time'][0], rdt.fill_value('time'))
stream_def_obj = self.pubsub_management.read_stream_definition(stream_def_id)
rdt = RecordDictionaryTool(stream_definition=stream_def_obj)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
granule = rdt.to_granule()
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
np.testing.assert_array_equal(rdt['temp'], np.arange(20))
def test_filter(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
filtered_stream_def_id = self.pubsub_management.create_stream_definition('filtered', parameter_dictionary_id=pdict_id, available_fields=['time', 'temp'])
self.addCleanup(self.pubsub_management.delete_stream_definition, filtered_stream_def_id)
rdt = RecordDictionaryTool(stream_definition_id=filtered_stream_def_id)
self.assertEquals(rdt._available_fields,['time','temp'])
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
with self.assertRaises(KeyError):
rdt['pressure'] = np.arange(20)
granule = rdt.to_granule()
rdt2 = RecordDictionaryTool.load_from_granule(granule)
self.assertEquals(rdt._available_fields, rdt2._available_fields)
self.assertEquals(rdt.fields, rdt2.fields)
for k,v in rdt.iteritems():
self.assertTrue(np.array_equal(rdt[k], rdt2[k]))
def test_rdt_param_funcs(self):
param_funcs = {
'identity' : {
'function_type' : PFT.PYTHON,
'owner' : 'ion_functions.data.interpolation',
'function' : 'identity',
'args':['x']
},
'ctd_tempwat' : {
'function_type' : PFT.PYTHON,
'owner' : 'ion_functions.data.ctd_functions',
'function' : 'ctd_sbe37im_tempwat',
'args' : ['t0']
},
'ctd_preswat' : {
'function_type' : PFT.PYTHON,
'owner' : 'ion_functions.data.ctd_functions',
'function' : 'ctd_sbe37im_preswat',
'args' : ["p0", "p_range_psia"]
},
'ctd_condwat' : {
'function_type' : PFT.PYTHON,
'owner' : 'ion_functions.data.ctd_functions',
'function' : 'ctd_sbe37im_condwat',
'args' : ['c0']
},
'ctd_pracsal' : {
'function_type' : PFT.PYTHON,
'owner' : 'ion_functions.data.ctd_functions',
'function' : 'ctd_pracsal',
'args' : ['c', 't', 'p']
},
'ctd_density' : {
'function_type' : PFT.PYTHON,
'owner' : 'ion_functions.data.ctd_functions',
'function' : 'ctd_density',
'args' : ['SP','t','p','lat','lon']
}
}
pfunc_ids = {}
for name, param_def in param_funcs.iteritems():
paramfunc = ParameterFunction(name, **param_def)
pf_id = self.dataset_management.create_parameter_function(paramfunc)
pfunc_ids[name] = pf_id
params = {
'time' : {
'parameter_type' : 'quantity',
'value_encoding' : 'float64',
'units' : 'seconds since 1900-01-01'
},
'temperature_counts' : {
'parameter_type' : 'quantity',
'value_encoding' : 'float32',
'units' : '1'
},
'pressure_counts' : {
'parameter_type' : 'quantity',
'value_encoding' : 'float32',
'units' : '1'
},
'conductivity_counts' : {
'parameter_type' : 'quantity',
'value_encoding' : 'float32',
'units' : '1'
},
'temperature' : {
'parameter_type' : 'function',
'parameter_function_id' : pfunc_ids['ctd_tempwat'],
'parameter_function_map' : { 't0' : 'temperature_counts'},
'value_encoding' : 'float32',
'units' : 'deg_C'
},
'pressure' : {
'parameter_type' : 'function',
'parameter_function_id' : pfunc_ids['ctd_preswat'],
'parameter_function_map' : {'p0' : 'pressure_counts', 'p_range_psia' : 679.34040721},
'value_encoding' : 'float32',
'units' : 'dbar'
},
'conductivity' : {
'parameter_type' : 'function',
'parameter_function_id' : pfunc_ids['ctd_condwat'],
'parameter_function_map' : {'c0' : 'conductivity_counts'},
'value_encoding' : 'float32',
'units' : 'Sm-1'
},
'salinity' : {
'parameter_type' : 'function',
'parameter_function_id' : pfunc_ids['ctd_pracsal'],
'parameter_function_map' : {'c' : 'conductivity', 't' : 'temperature', 'p' : 'pressure'},
'value_encoding' : 'float32',
'units' : '1'
},
'density' : {
'parameter_type' : 'function',
'parameter_function_id' : pfunc_ids['ctd_density'],
'parameter_function_map' : {
'SP' : 'salinity',
't' : 'temperature',
'p' : 'pressure',
'lat' : 'lat',
'lon' : 'lon'
},
'value_encoding' : 'float32',
'units' : 'kg m-1'
},
'lat' : {
'parameter_type' : 'sparse',
'value_encoding' : 'float32',
'units' : 'degrees_north'
},
'lon' : {
'parameter_type' : 'sparse',
'value_encoding' : 'float32',
'units' : 'degrees_east'
}
}
param_dict = {}
for name, param in params.iteritems():
pcontext = ParameterContext(name, **param)
param_id = self.dataset_management.create_parameter(pcontext)
param_dict[name] = param_id
pdict_id = self.dataset_management.create_parameter_dictionary('ctd_test', param_dict.values(), 'time')
stream_def_id = self.pubsub_management.create_stream_definition('ctd_test', parameter_dictionary_id=pdict_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temperature_counts'] = [280000]
rdt['conductivity_counts'] = [100000]
rdt['pressure_counts'] = [2789]
rdt['lat'] = [45]
rdt['lon'] = [-71]
np.testing.assert_allclose(rdt['density'], np.array([1001.00543606]))
def test_rdt_lookup(self):
rdt = self.create_lookup_rdt()
self.assertTrue('offset_a' in rdt.lookup_values())
self.assertFalse('offset_b' in rdt.lookup_values())
rdt['time'] = [0]
rdt['temp'] = [10.0]
rdt['offset_a'] = [2.0]
self.assertEquals(rdt['offset_b'], None)
self.assertEquals(rdt.lookup_values(), ['offset_a'])
np.testing.assert_array_almost_equal(rdt['calibrated'], np.array([12.0]))
svm = StoredValueManager(self.container)
svm.stored_value_cas('coefficient_document', {'offset_b':2.0})
svm.stored_value_cas("GA03FLMA-RI001-13-CTDMOG999_OFFSETC", {'offset_c':3.0})
rdt.fetch_lookup_values()
np.testing.assert_array_equal(rdt['offset_b'], np.array([2.0]))
np.testing.assert_array_equal(rdt['calibrated_b'], np.array([14.0]))
np.testing.assert_array_equal(rdt['offset_c'], np.array([3.0]))
def create_rdt(self):
contexts, pfuncs = self.create_pfuncs()
context_ids = list(contexts.itervalues())
pdict_id = self.dataset_management.create_parameter_dictionary(name='functional_pdict', parameter_context_ids=context_ids, temporal_context='test_TIME')
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
stream_def_id = self.pubsub_management.create_stream_definition('functional', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
return rdt
def create_lookup_rdt(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsub_management.create_stream_definition('lookup', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':"GA03FLMA-RI001-13-CTDMOG999"})
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
return rdt
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext(name='TIME',
parameter_type='quantity',
value_encoding='float64',
units='seconds since 1900-01-01')
t_ctxt_id = self.dataset_management.create_parameter(t_ctxt)
contexts['TIME'] = t_ctxt_id
lat_ctxt = ParameterContext(name='LAT',
parameter_type="sparse",
value_encoding='float32',
units='degrees_north')
lat_ctxt_id = self.dataset_management.create_parameter(lat_ctxt)
contexts['LAT'] = lat_ctxt_id
lon_ctxt = ParameterContext(name='LON',
parameter_type='sparse',
value_encoding='float32',
units='degrees_east')
lon_ctxt_id = self.dataset_management.create_parameter(lon_ctxt)
contexts['LON'] = lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(name='TEMPWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='deg_C')
temp_ctxt_id = self.dataset_management.create_parameter(temp_ctxt)
contexts['TEMPWAT_L0'] = temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(name='CONDWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='S m-1')
cond_ctxt_id = self.dataset_management.create_parameter(cond_ctxt)
contexts['CONDWAT_L0'] = cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(name='PRESWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='dbar')
press_ctxt_id = self.dataset_management.create_parameter(press_ctxt)
contexts['PRESWAT_L0'] = press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = ParameterFunction(name='TEMPWAT_L1',
function_type=PFT.NUMEXPR,
function=tl1_func,
args=['T'])
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['TEMPWAT_L1'] = expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
tempL1_ctxt = ParameterContext(name='TEMPWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=tl1_pmap,
value_encoding='float32',
units='deg_C')
tempL1_ctxt_id = self.dataset_management.create_parameter(tempL1_ctxt)
contexts['TEMPWAT_L1'] = tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = ParameterFunction(name='CONDWAT_L1',
function_type=PFT.NUMEXPR,
function=cl1_func,
args=['C'])
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['CONDWAT_L1'] = expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
condL1_ctxt = ParameterContext(name='CONDWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=cl1_pmap,
value_encoding='float32',
units='S m-1')
condL1_ctxt_id = self.dataset_management.create_parameter(condL1_ctxt)
contexts['CONDWAT_L1'] = condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = ParameterFunction(name='PRESWAT_L1',function=pl1_func,function_type=PFT.NUMEXPR,args=['P','p_range'])
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['PRESWAT_L1'] = expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
presL1_ctxt = ParameterContext(name='PRESWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=pl1_pmap,
value_encoding='float32',
units='S m-1')
presL1_ctxt_id = self.dataset_management.create_parameter(presL1_ctxt)
contexts['PRESWAT_L1'] = presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = ParameterFunction(name='PRACSAL',function_type=PFT.PYTHON,function=sal_func,owner=owner,args=sal_arglist)
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['PRACSAL'] = expr_id
c10_f = ParameterFunction(name='c10', function_type=PFT.NUMEXPR, function='C*10', args=['C'])
expr_id = self.dataset_management.create_parameter_function(c10_f)
c10 = ParameterContext(name='c10',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map={'C':'CONDWAT_L1'},
value_encoding='float32',
units='1')
c10_id = self.dataset_management.create_parameter(c10)
contexts['c10'] = c10_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {'C': 'c10', 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
sal_ctxt = ParameterContext(name='PRACSAL',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=sal_pmap,
value_encoding='float32',
units='g kg-1')
sal_ctxt_id = self.dataset_management.create_parameter(sal_ctxt)
contexts['PRACSAL'] = sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT'])
cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = CoverageParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name='DENSITY', parameter_context=dens_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id)
contexts['DENSITY'] = dens_ctxt_id
return contexts, funcs
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
def test_raw_stream_integration(self):
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here to run as a script (don't forget the imports of course!)
#-----------------------------
# Create some service clients...
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(
node=cc.node)
dataset_management_service = DatasetManagementServiceClient(
node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
# declare some handy variables
datastore_name = 'test_dm_integration'
###
### In the beginning there was one stream definitions...
###
# create a stream definition for the data from the ctd simulator
raw_ctd_stream_def = SBE37_RAW_stream_definition()
raw_ctd_stream_def_id = pubsub_management_service.create_stream_definition(
container=raw_ctd_stream_def, name='Simulated RAW CTD data')
###
### And two process definitions...
###
# one for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.raw_stream_publisher',
'class': 'RawStreamPublisher'
}
raw_ctd_sim_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
#---------------------------
# Set up ingestion - this is an operator concern - not done by SA in a deployed system
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,
datastore_profile='SCIDATA'),
number_of_workers=1)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
#---------------------------
# Set up the producer (CTD Simulator)
#---------------------------
# Create the stream
raw_ctd_stream_id = pubsub_management_service.create_stream(
stream_definition_id=raw_ctd_stream_def_id)
# Set up the datasets
raw_ctd_dataset_id = dataset_management_service.create_dataset(
stream_id=raw_ctd_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of this dataset
raw_ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=raw_ctd_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
# Start the ctd simulator to produce some data
configuration = {
'process': {
'stream_id': raw_ctd_stream_id,
}
}
raw_sim_pid = process_dispatcher.schedule_process(
process_definition_id=raw_ctd_sim_procdef_id,
configuration=configuration)
###
### Make a subscriber in the test to listen for salinity data
###
raw_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery([
raw_ctd_stream_id,
]),
exchange_name='raw_test',
name="test raw subscription",
)
# this is okay - even in cei mode!
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process,
node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn('Raw data received!')
results.append(message)
if len(results) > 3:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(
exchange_name='raw_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
pubsub_management_service.activate_subscription(
subscription_id=raw_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=10))
# stop the flow parse the messages...
process_dispatcher.cancel_process(
raw_sim_pid
) # kill the ctd simulator process - that is enough data
gevent.sleep(1)
for message in results:
sha1 = message.identifiables['stream_encoding'].sha1
data = message.identifiables['data_stream'].values
filename = FileSystem.get_hierarchical_url(FS.CACHE, sha1, ".raw")
with open(filename, 'r') as f:
assertions(data == f.read())
class TestDMEnd2End(IonIntegrationTestCase):
def setUp(self): # Love the non pep-8 convention
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.ingestion_management = IngestionManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.pids = []
self.event = Event()
self.exchange_space_name = 'test_granules'
self.exchange_point_name = 'science_data'
self.i = 0
self.purge_queues()
self.queue_buffer = []
self.streams = []
self.addCleanup(self.stop_all_ingestion)
def purge_queues(self):
xn = self.container.ex_manager.create_xn_queue(
'science_granule_ingestion')
xn.purge()
def tearDown(self):
self.purge_queues()
for pid in self.pids:
self.container.proc_manager.terminate_process(pid)
IngestionManagementIntTest.clean_subscriptions()
for queue in self.queue_buffer:
if isinstance(queue, ExchangeNameQueue):
queue.delete()
elif isinstance(queue, str):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
#--------------------------------------------------------------------------------
# Helper/Utility methods
#--------------------------------------------------------------------------------
def create_dataset(self, parameter_dict_id=''):
'''
Creates a time-series dataset
'''
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
if not parameter_dict_id:
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
dataset_id = self.dataset_management.create_dataset(
'test_dataset_%i' % self.i,
parameter_dictionary_id=parameter_dict_id,
spatial_domain=sdom,
temporal_domain=tdom)
return dataset_id
def get_datastore(self, dataset_id):
'''
Gets an instance of the datastore
This method is primarily used to defeat a bug where integration tests in multiple containers may sometimes
delete a CouchDB datastore and the other containers are unaware of the new state of the datastore.
'''
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def get_ingestion_config(self):
'''
Grab the ingestion configuration from the resource registry
'''
# The ingestion configuration should have been created by the bootstrap service
# which is configured through r2deploy.yml
ingest_configs, _ = self.resource_registry.find_resources(
restype=RT.IngestionConfiguration, id_only=True)
return ingest_configs[0]
def launch_producer(self, stream_id=''):
'''
Launch the producer
'''
pid = self.container.spawn_process(
'better_data_producer', 'ion.processes.data.example_data_producer',
'BetterDataProducer', {'process': {
'stream_id': stream_id
}})
self.pids.append(pid)
def make_simple_dataset(self):
'''
Makes a stream, a stream definition and a dataset, the essentials for most of these tests
'''
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition(
'ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream(
'ctd stream %i' % self.i,
'xp1',
stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
self.get_datastore(dataset_id)
self.i += 1
return stream_id, route, stream_def_id, dataset_id
def publish_hifi(self, stream_id, stream_route, offset=0):
'''
Publish deterministic data
'''
pub = StandaloneStreamPublisher(stream_id, stream_route)
stream_def = self.pubsub_management.read_stream_definition(
stream_id=stream_id)
stream_def_id = stream_def._id
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10) + (offset * 10)
rdt['temp'] = np.arange(10) + (offset * 10)
pub.publish(rdt.to_granule())
def publish_fake_data(self, stream_id, route):
'''
Make four granules
'''
for i in xrange(4):
self.publish_hifi(stream_id, route, i)
def start_ingestion(self, stream_id, dataset_id):
'''
Starts ingestion/persistence for a given dataset
'''
ingest_config_id = self.get_ingestion_config()
self.ingestion_management.persist_data_stream(
stream_id=stream_id,
ingestion_configuration_id=ingest_config_id,
dataset_id=dataset_id)
def stop_ingestion(self, stream_id):
ingest_config_id = self.get_ingestion_config()
self.ingestion_management.unpersist_data_stream(
stream_id=stream_id, ingestion_configuration_id=ingest_config_id)
def stop_all_ingestion(self):
try:
[self.stop_ingestion(sid) for sid in self.streams]
except:
pass
def validate_granule_subscription(self, msg, route, stream_id):
'''
Validation for granule format
'''
if msg == {}:
return
rdt = RecordDictionaryTool.load_from_granule(msg)
log.info('%s', rdt.pretty_print())
self.assertIsInstance(
msg, Granule, 'Message is improperly formatted. (%s)' % type(msg))
self.event.set()
def wait_until_we_have_enough_granules(self, dataset_id='', data_size=40):
'''
Loops until there is a sufficient amount of data in the dataset
'''
done = False
with gevent.Timeout(40):
while not done:
extents = self.dataset_management.dataset_extents(
dataset_id, 'time')[0]
granule = self.data_retriever.retrieve_last_data_points(
dataset_id, 1)
rdt = RecordDictionaryTool.load_from_granule(granule)
if rdt['time'] and rdt['time'][0] != rdt._pdict.get_context(
'time').fill_value and extents >= data_size:
done = True
else:
gevent.sleep(0.2)
#--------------------------------------------------------------------------------
# Test Methods
#--------------------------------------------------------------------------------
@attr('SMOKE')
def test_dm_end_2_end(self):
#--------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
#--------------------------------------------------------------------------------
self.event.clear()
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(
self.dataset_management.create_parameter_context(
'binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(
self.dataset_management.create_parameter_context(
'records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'replay_pdict',
parameter_context_ids=context_ids,
temporal_context='time')
stream_definition = self.pubsub_management.create_stream_definition(
'ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream(
'producer',
exchange_point=self.exchange_point_name,
stream_definition_id=stream_definition)
#--------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
#--------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(
stream_id=stream_id,
ingestion_configuration_id=ingest_config_id,
dataset_id=dataset_id)
#--------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
#--------------------------------------------------------------------------------
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id, 40)
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),
'%s' % rdt['time'][:])
self.assertTrue((rdt['binary'][:10] == np.array(['hi'] * 10,
dtype='object')).all())
#--------------------------------------------------------------------------------
# Now to try the streamed approach
#--------------------------------------------------------------------------------
replay_stream_id, replay_route = self.pubsub_management.create_stream(
'replay_out',
exchange_point=self.exchange_point_name,
stream_definition_id=stream_definition)
self.replay_id, process_id = self.data_retriever.define_replay(
dataset_id=dataset_id, stream_id=replay_stream_id)
log.info('Process ID: %s', process_id)
replay_client = ReplayClient(process_id)
#--------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
#--------------------------------------------------------------------------------
xp = self.container.ex_manager.create_xp(self.exchange_point_name)
subscriber = StandaloneStreamSubscriber(
self.exchange_space_name, self.validate_granule_subscription)
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
self.data_retriever.start_replay_agent(self.replay_id)
self.assertTrue(replay_client.await_agent_ready(5),
'The process never launched')
replay_client.start_replay()
self.assertTrue(self.event.wait(10))
subscriber.stop()
self.data_retriever.cancel_replay_agent(self.replay_id)
#--------------------------------------------------------------------------------
# Test the slicing capabilities
#--------------------------------------------------------------------------------
granule = self.data_retriever.retrieve(dataset_id=dataset_id,
query={'tdoa': slice(0, 5)})
rdt = RecordDictionaryTool.load_from_granule(granule)
b = rdt['time'] == np.arange(5)
self.assertTrue(b.all() if not isinstance(b, bool) else b)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
@unittest.skip('Doesnt work')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_replay_pause(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(
self.dataset_management.create_parameter_context(
'binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(
self.dataset_management.create_parameter_context(
'records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'replay_pdict',
parameter_context_ids=context_ids,
temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition(
'replay_stream', parameter_dictionary_id=pdict_id)
replay_stream, replay_route = self.pubsub_management.create_stream(
'replay', 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
scov = DatasetManagementService._get_coverage(dataset_id)
bb = CoverageCraft(scov)
bb.rdt['time'] = np.arange(100)
bb.rdt['temp'] = np.random.random(100) + 30
bb.sync_with_granule()
DatasetManagementService._persist_coverage(
dataset_id,
bb.coverage) # This invalidates it for multi-host configurations
# Set up the subscriber to verify the data
subscriber = StandaloneStreamSubscriber(
self.exchange_space_name, self.validate_granule_subscription)
xp = self.container.ex_manager.create_xp('xp1')
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
# Set up the replay agent and the client wrapper
# 1) Define the Replay (dataset and stream to publish on)
self.replay_id, process_id = self.data_retriever.define_replay(
dataset_id=dataset_id, stream_id=replay_stream)
# 2) Make a client to the interact with the process (optionall provide it a process to bind with)
replay_client = ReplayClient(process_id)
# 3) Start the agent (launch the process)
self.data_retriever.start_replay_agent(self.replay_id)
# 4) Start replaying...
replay_client.start_replay()
# Wait till we get some granules
self.assertTrue(self.event.wait(5))
# We got granules, pause the replay, clear the queue and allow the process to finish consuming
replay_client.pause_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure there's no remaining messages being consumed
self.assertFalse(self.event.wait(1))
# Resume the replay and wait until we start getting granules again
replay_client.resume_replay()
self.assertTrue(self.event.wait(5))
# Stop the replay, clear the queues
replay_client.stop_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure that it did indeed stop
self.assertFalse(self.event.wait(1))
subscriber.stop()
def test_retrieve_and_transform(self):
# Make a simple dataset and start ingestion, pretty standard stuff.
ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset(
)
self.start_ingestion(ctd_stream_id, dataset_id)
# Stream definition for the salinity data
salinity_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
sal_stream_def_id = self.pubsub_management.create_stream_definition(
'sal data', parameter_dictionary_id=salinity_pdict_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.random.randn(10) * 10 + 30
rdt['conductivity'] = np.random.randn(10) * 2 + 10
rdt['pressure'] = np.random.randn(10) * 1 + 12
publisher = StandaloneStreamPublisher(ctd_stream_id, route)
publisher.publish(rdt.to_granule())
rdt['time'] = np.arange(10, 20)
publisher.publish(rdt.to_granule())
self.wait_until_we_have_enough_granules(dataset_id, 20)
granule = self.data_retriever.retrieve(
dataset_id,
None,
None,
'ion.processes.data.transforms.ctd.ctd_L2_salinity',
'CTDL2SalinityTransformAlgorithm',
kwargs=dict(params=sal_stream_def_id))
rdt = RecordDictionaryTool.load_from_granule(granule)
for i in rdt['salinity']:
self.assertNotEquals(i, 0)
self.streams.append(ctd_stream_id)
self.stop_ingestion(ctd_stream_id)
def test_last_granule(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset(
)
self.start_ingestion(stream_id, dataset_id)
self.publish_hifi(stream_id, route, 0)
self.publish_hifi(stream_id, route, 1)
self.wait_until_we_have_enough_granules(dataset_id,
20) # I just need two
success = False
def verifier():
replay_granule = self.data_retriever.retrieve_last_data_points(
dataset_id, 10)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(10) + 10
if not isinstance(comp, bool):
return comp.all()
return False
success = poll(verifier)
self.assertTrue(success)
success = False
def verify_points():
replay_granule = self.data_retriever.retrieve_last_data_points(
dataset_id, 5)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(15, 20)
if not isinstance(comp, bool):
return comp.all()
return False
success = poll(verify_points)
self.assertTrue(success)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
def test_replay_with_parameters(self):
#--------------------------------------------------------------------------------
# Create the configurations and the dataset
#--------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(
self.dataset_management.create_parameter_context(
'binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(
self.dataset_management.create_parameter_context(
'records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'replay_pdict',
parameter_context_ids=context_ids,
temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition(
'replay_stream', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream(
'replay_with_params',
exchange_point=self.exchange_point_name,
stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(
stream_id=stream_id,
ingestion_configuration_id=config_id,
dataset_id=dataset_id)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id)
es.start()
self.addCleanup(es.stop)
self.publish_fake_data(stream_id, route)
self.assertTrue(dataset_modified.wait(30))
query = {
'start_time': 0 - 2208988800,
'end_time': 20 - 2208988800,
'stride_time': 2,
'parameters': ['time', 'temp']
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,
query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
comp = np.arange(0, 20, 2) == rdt['time']
self.assertTrue(comp.all(), '%s' % rdt.pretty_print())
self.assertEquals(set(rdt.iterkeys()), set(['time', 'temp']))
extents = self.dataset_management.dataset_extents(
dataset_id=dataset_id, parameters=['time', 'temp'])
self.assertTrue(extents['time'] >= 20)
self.assertTrue(extents['temp'] >= 20)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
def test_repersist_data(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset(
)
self.start_ingestion(stream_id, dataset_id)
self.publish_hifi(stream_id, route, 0)
self.publish_hifi(stream_id, route, 1)
self.wait_until_we_have_enough_granules(dataset_id, 20)
config_id = self.get_ingestion_config()
self.ingestion_management.unpersist_data_stream(
stream_id=stream_id, ingestion_configuration_id=config_id)
self.ingestion_management.persist_data_stream(
stream_id=stream_id,
ingestion_configuration_id=config_id,
dataset_id=dataset_id)
self.publish_hifi(stream_id, route, 2)
self.publish_hifi(stream_id, route, 3)
self.wait_until_we_have_enough_granules(dataset_id, 40)
success = False
with gevent.timeout.Timeout(5):
while not success:
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(0, 40)
if not isinstance(comp, bool):
success = comp.all()
gevent.sleep(1)
self.assertTrue(success)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
@attr('LOCOINT')
@unittest.skipIf(os.getenv(
'CEI_LAUNCH_TEST', False
), 'Host requires file-system access to coverage files, CEI mode does not support.'
)
def test_correct_time(self):
# There are 2208988800 seconds between Jan 1 1900 and Jan 1 1970, i.e.
# the conversion factor between unix and NTP time
unix_now = np.floor(time.time())
ntp_now = unix_now + 2208988800
unix_ago = unix_now - 20
ntp_ago = unix_ago + 2208988800
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset(
)
coverage = DatasetManagementService._get_coverage(dataset_id)
coverage.insert_timesteps(20)
coverage.set_parameter_values('time', np.arange(ntp_ago, ntp_now))
temporal_bounds = self.dataset_management.dataset_temporal_bounds(
dataset_id)
self.assertTrue(np.abs(temporal_bounds[0] - unix_ago) < 2)
self.assertTrue(np.abs(temporal_bounds[1] - unix_now) < 2)
@attr('LOCOINT')
@unittest.skipIf(os.getenv(
'CEI_LAUNCH_TEST', False
), 'Host requires file-system access to coverage files, CEI mode does not support.'
)
def test_empty_coverage_time(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset(
)
coverage = DatasetManagementService._get_coverage(dataset_id)
temporal_bounds = self.dataset_management.dataset_temporal_bounds(
dataset_id)
self.assertEquals([coverage.get_parameter_context('time').fill_value] *
2, temporal_bounds)
@attr('LOCOINT')
@unittest.skipIf(os.getenv(
'CEI_LAUNCH_TEST', False
), 'Host requires file-system access to coverage files, CEI mode does not support.'
)
def test_out_of_band_retrieve(self):
# Setup the environemnt
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset(
)
self.start_ingestion(stream_id, dataset_id)
# Fill the dataset
self.publish_fake_data(stream_id, route)
self.wait_until_we_have_enough_granules(dataset_id, 40)
# Retrieve the data
granule = DataRetrieverService.retrieve_oob(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assertTrue((rdt['time'] == np.arange(40)).all())
@attr('LOCOINT')
@unittest.skipIf(os.getenv(
'CEI_LAUNCH_TEST', False
), 'Host requires file-system access to coverage files, CEI mode does not support.'
)
def test_retrieve_cache(self):
DataRetrieverService._refresh_interval = 1
datasets = [self.make_simple_dataset() for i in xrange(10)]
for stream_id, route, stream_def_id, dataset_id in datasets:
coverage = DatasetManagementService._get_coverage(dataset_id)
coverage.insert_timesteps(10)
coverage.set_parameter_values('time', np.arange(10))
coverage.set_parameter_values('temp', np.arange(10))
# Verify cache hit and refresh
dataset_ids = [i[3] for i in datasets]
self.assertTrue(
dataset_ids[0] not in DataRetrieverService._retrieve_cache)
DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache
cov, age = DataRetrieverService._retrieve_cache[dataset_ids[0]]
# Verify that it was hit and it's now in there
self.assertTrue(dataset_ids[0] in DataRetrieverService._retrieve_cache)
gevent.sleep(DataRetrieverService._refresh_interval + 0.2)
DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache
cov, age2 = DataRetrieverService._retrieve_cache[dataset_ids[0]]
self.assertTrue(age2 != age)
for dataset_id in dataset_ids:
DataRetrieverService._get_coverage(dataset_id)
self.assertTrue(
dataset_ids[0] not in DataRetrieverService._retrieve_cache)
stream_id, route, stream_def, dataset_id = datasets[0]
self.start_ingestion(stream_id, dataset_id)
DataRetrieverService._get_coverage(dataset_id)
self.assertTrue(dataset_id in DataRetrieverService._retrieve_cache)
DataRetrieverService._refresh_interval = 100
self.publish_hifi(stream_id, route, 1)
self.wait_until_we_have_enough_granules(dataset_id, data_size=20)
event = gevent.event.Event()
with gevent.Timeout(20):
while not event.wait(0.1):
if dataset_id not in DataRetrieverService._retrieve_cache:
event.set()
self.assertTrue(event.is_set())
@unittest.skip('Outdated due to ingestion retry')
@attr('LOCOINT')
@unittest.skipIf(os.getenv(
'CEI_LAUNCH_TEST', False
), 'Host requires file-system access to coverage files, CEI mode does not support.'
)
def test_ingestion_failover(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset(
)
self.start_ingestion(stream_id, dataset_id)
event = Event()
def cb(*args, **kwargs):
event.set()
sub = EventSubscriber(event_type="ExceptionEvent",
callback=cb,
origin="stream_exception")
sub.start()
self.publish_fake_data(stream_id, route)
self.wait_until_we_have_enough_granules(dataset_id, 40)
file_path = DatasetManagementService._get_coverage_path(dataset_id)
master_file = os.path.join(file_path, '%s_master.hdf5' % dataset_id)
with open(master_file, 'w') as f:
f.write('this will crash HDF')
self.publish_hifi(stream_id, route, 5)
self.assertTrue(event.wait(10))
sub.stop()
class PubSubIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1",
description="Sample Stream 1 Description")
self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2",
description="Sample Stream 2 Description")
# Make a subscription to two input streams
exchange_name = "a_queue"
query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id])
self.ctd_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleSubscription",
"Sample Subscription Description")
# Make a subscription to all streams on an exchange point
exchange_name = "another_queue"
query = ExchangeQuery()
self.exchange_subscription_id = self.pubsub_cli.create_subscription(query,
exchange_name,
"SampleExchangeSubscription",
"Sample Exchange Subscription Description")
pid = self.container.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = self.container.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.container.node)
self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream1_id)
self.ctd_stream2_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream2_id)
# Cheat and use the cc as the process - I don't think it is used for anything...
self.stream_subscriber = StreamSubscriberRegistrar(process=dummy_process, node=self.container.node)
def tearDown(self):
self.pubsub_cli.delete_subscription(self.ctd_subscription_id)
self.pubsub_cli.delete_subscription(self.exchange_subscription_id)
self.pubsub_cli.delete_stream(self.ctd_stream1_id)
self.pubsub_cli.delete_stream(self.ctd_stream2_id)
self._stop_container()
def test_bind_stream_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='a_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.ctd_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(q.get(timeout=5), 'message2')
self.assertTrue(q.empty())
subscriber.stop()
def test_bind_exchange_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='another_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.exchange_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(q.get(timeout=5), 'message2')
self.assertTrue(q.empty())
subscriber.stop()
def test_unbind_stream_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='a_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.ctd_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.pubsub_cli.deactivate_subscription(self.ctd_subscription_id)
self.ctd_stream2_publisher.publish('message2')
p = None
with self.assertRaises(gevent.queue.Empty) as cm:
p = q.get(timeout=1)
subscriber.stop()
ex = cm.exception
self.assertEqual(str(ex), '')
self.assertEqual(p, None)
def test_unbind_exchange_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='another_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.exchange_subscription_id)
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.pubsub_cli.deactivate_subscription(self.exchange_subscription_id)
self.ctd_stream2_publisher.publish('message2')
p = None
with self.assertRaises(gevent.queue.Empty) as cm:
p = q.get(timeout=1)
subscriber.stop()
ex = cm.exception
self.assertEqual(str(ex), '')
self.assertEqual(p, None)
def test_update_stream_subscription(self):
q = gevent.queue.Queue()
def message_received(message, headers):
q.put(message)
subscriber = self.stream_subscriber.create_subscriber(exchange_name='a_queue', callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.ctd_subscription_id)
# Both publishers are received by the subscriber
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(q.get(timeout=5), 'message2')
self.assertTrue(q.empty())
# Update the subscription by removing a stream...
subscription = self.pubsub_cli.read_subscription(self.ctd_subscription_id)
stream_ids = list(subscription.query.stream_ids)
stream_ids.remove(self.ctd_stream2_id)
self.pubsub_cli.update_subscription(
subscription_id=subscription._id,
query=StreamQuery(stream_ids=stream_ids)
)
# Stream 2 is no longer received
self.ctd_stream2_publisher.publish('message2')
p = None
with self.assertRaises(gevent.queue.Empty) as cm:
p = q.get(timeout=1)
ex = cm.exception
self.assertEqual(str(ex), '')
self.assertEqual(p, None)
# Stream 1 is as before
self.ctd_stream1_publisher.publish('message1')
self.assertEqual(q.get(timeout=5), 'message1')
self.assertTrue(q.empty())
# Now swith the active streams...
# Update the subscription by removing a stream...
self.pubsub_cli.update_subscription(
subscription_id=self.ctd_subscription_id,
query=StreamQuery([self.ctd_stream2_id])
)
# Stream 1 is no longer received
self.ctd_stream1_publisher.publish('message1')
p = None
with self.assertRaises(gevent.queue.Empty) as cm:
p = q.get(timeout=1)
ex = cm.exception
self.assertEqual(str(ex), '')
self.assertEqual(p, None)
# Stream 2 is received
self.ctd_stream2_publisher.publish('message2')
self.assertEqual(q.get(timeout=5), 'message2')
self.assertTrue(q.empty())
subscriber.stop()
def test_find_stream_definition(self):
definition = SBE37_CDM_stream_definition()
definition_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=definition_id)
res_id = self.pubsub_cli.find_stream_definition(stream_id=stream_id, id_only=True)
self.assertTrue(res_id==definition_id, 'The returned id did not match the definition_id')
res_obj = self.pubsub_cli.find_stream_definition(stream_id=stream_id, id_only=False)
self.assertTrue(isinstance(res_obj.container, StreamDefinitionContainer),
'The container object is not a stream definition.')
def test_strem_def_not_found(self):
with self.assertRaises(NotFound):
self.pubsub_cli.find_stream_definition(stream_id='nonexistent')
definition = SBE37_CDM_stream_definition()
definition_id = self.pubsub_cli.create_stream_definition(container=definition)
with self.assertRaises(NotFound):
self.pubsub_cli.find_stream_definition(stream_id='nonexistent')
stream_id = self.pubsub_cli.create_stream()
with self.assertRaises(NotFound):
self.pubsub_cli.find_stream_definition(stream_id=stream_id)
@unittest.skip("Nothing to test")
def test_bind_already_bound_subscription(self):
pass
@unittest.skip("Nothing to test")
def test_unbind_unbound_subscription(self):
pass
def run_external_transform(self):
'''
This example script illustrates how a transform can interact with the an outside process (very basic)
it launches an external_transform example which uses the operating system command 'bc' to add 1 to the input
Producer -> A -> 'FS.TEMP/transform_output'
A is an external transform that spawns an OS process to increment the input by 1
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = ProcessDefinition(
name='external_transform_definition')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ExternalTransform'
process_definition_id = procd_cli.create_process_definition(
process_definition=process_definition)
#-------------------------------
# Streams
#-------------------------------
input_stream_id = pubsub_cli.create_stream(name='input_stream',
original=True)
#-------------------------------
# Subscription
#-------------------------------
query = StreamQuery(stream_ids=[input_stream_id])
input_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='input_queue')
#-------------------------------
# Launch Transform
#-------------------------------
transform_id = tms_cli.create_transform(
name='external_transform',
in_subscription_id=input_subscription_id,
process_definition_id=process_definition_id,
configuration={})
tms_cli.activate_transform(transform_id)
#-------------------------------
# Launch Producer
#-------------------------------
id_p = self.container.spawn_process(
'myproducer', 'ion.processes.data.transforms.transform_example',
'TransformExampleProducer', {
'process': {
'type': 'stream_process',
'publish_streams': {
'out_stream': input_stream_id
}
},
'stream_producer': {
'interval': 4000
}
})
self.container.proc_manager.procs[id_p].start()
class PubsubManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.pdicts = {}
self.queue_cleanup = list()
self.exchange_cleanup = list()
def tearDown(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
for exchange in self.exchange_cleanup:
xp = self.container.ex_manager.create_xp(exchange)
xp.delete()
def test_stream_def_crud(self):
# Test Creation
pdict = DatasetManagementService.get_parameter_dictionary_by_name(
'ctd_parsed_param_dict')
stream_definition_id = self.pubsub_management.create_stream_definition(
'ctd parsed', parameter_dictionary_id=pdict.identifier)
# Make sure there is an assoc
self.assertTrue(
self.resource_registry.find_associations(
subject=stream_definition_id,
predicate=PRED.hasParameterDictionary,
object=pdict.identifier,
id_only=True))
# Test Reading
stream_definition = self.pubsub_management.read_stream_definition(
stream_definition_id)
self.assertTrue(
PubsubManagementService._compare_pdicts(
pdict.dump(), stream_definition.parameter_dictionary))
# Test Deleting
self.pubsub_management.delete_stream_definition(stream_definition_id)
self.assertFalse(
self.resource_registry.find_associations(
subject=stream_definition_id,
predicate=PRED.hasParameterDictionary,
object=pdict.identifier,
id_only=True))
# Test comparisons
in_stream_definition_id = self.pubsub_management.create_stream_definition(
'L0 products',
parameter_dictionary_id=pdict.identifier,
available_fields=['time', 'temp', 'conductivity', 'pressure'])
self.addCleanup(self.pubsub_management.delete_stream_definition,
in_stream_definition_id)
out_stream_definition_id = in_stream_definition_id
self.assertTrue(
self.pubsub_management.compare_stream_definition(
in_stream_definition_id, out_stream_definition_id))
self.assertTrue(
self.pubsub_management.compatible_stream_definitions(
in_stream_definition_id, out_stream_definition_id))
out_stream_definition_id = self.pubsub_management.create_stream_definition(
'L2 Products',
parameter_dictionary_id=pdict.identifier,
available_fields=['time', 'salinity', 'density'])
self.addCleanup(self.pubsub_management.delete_stream_definition,
out_stream_definition_id)
self.assertFalse(
self.pubsub_management.compare_stream_definition(
in_stream_definition_id, out_stream_definition_id))
self.assertTrue(
self.pubsub_management.compatible_stream_definitions(
in_stream_definition_id, out_stream_definition_id))
def test_validate_stream_defs(self):
#test no input
incoming_pdict_id = self._get_pdict(
['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'])
outgoing_pdict_id = self._get_pdict(
['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'])
available_fields_in = []
available_fields_out = []
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_0',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_0',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertFalse(result)
#test input with no output
incoming_pdict_id = self._get_pdict(
['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'])
outgoing_pdict_id = self._get_pdict(
['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'])
available_fields_in = [
'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'
]
available_fields_out = []
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_1',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_1',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertTrue(result)
#test available field missing parameter context definition -- missing PRESWAT_L0
incoming_pdict_id = self._get_pdict(
['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0'])
outgoing_pdict_id = self._get_pdict(
['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'])
available_fields_in = [
'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'
]
available_fields_out = ['DENSITY']
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_2',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_2',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertFalse(result)
#test l1 from l0
incoming_pdict_id = self._get_pdict(
['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'])
outgoing_pdict_id = self._get_pdict(
['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'])
available_fields_in = [
'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'
]
available_fields_out = ['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_3',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_3',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertTrue(result)
#test l2 from l0
incoming_pdict_id = self._get_pdict(
['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'])
outgoing_pdict_id = self._get_pdict(
['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1', 'DENSITY', 'PRACSAL'])
available_fields_in = [
'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'
]
available_fields_out = ['DENSITY', 'PRACSAL']
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_4',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_4',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertTrue(result)
#test Ln from L0
incoming_pdict_id = self._get_pdict(
['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'])
outgoing_pdict_id = self._get_pdict(
['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'])
available_fields_in = [
'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'
]
available_fields_out = [
'DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'
]
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_5',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_5',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertTrue(result)
#test L2 from L1
incoming_pdict_id = self._get_pdict(
['TIME', 'LAT', 'LON', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'])
outgoing_pdict_id = self._get_pdict(
['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'])
available_fields_in = [
'TIME', 'LAT', 'LON', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'
]
available_fields_out = ['DENSITY', 'PRACSAL']
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_6',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_6',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertTrue(result)
#test L1 from L0 missing L0
incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON'])
outgoing_pdict_id = self._get_pdict(
['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'])
available_fields_in = ['TIME', 'LAT', 'LON']
available_fields_out = ['DENSITY', 'PRACSAL']
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_7',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_7',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertFalse(result)
#test L2 from L0 missing L0
incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON'])
outgoing_pdict_id = self._get_pdict(
['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'])
available_fields_in = ['TIME', 'LAT', 'LON']
available_fields_out = ['DENSITY', 'PRACSAL']
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_8',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_8',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertFalse(result)
#test L2 from L0 missing L1
incoming_pdict_id = self._get_pdict(
['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'])
outgoing_pdict_id = self._get_pdict(['DENSITY', 'PRACSAL'])
available_fields_in = [
'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'
]
available_fields_out = ['DENSITY', 'PRACSAL']
incoming_stream_def_id = self.pubsub_management.create_stream_definition(
'in_sd_9',
parameter_dictionary_id=incoming_pdict_id,
available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition(
'out_sd_9',
parameter_dictionary_id=outgoing_pdict_id,
available_fields=available_fields_out)
result = self.pubsub_management.validate_stream_defs(
incoming_stream_def_id, outgoing_stream_def_id)
self.assertFalse(result)
def publish_on_stream(self, stream_id, msg):
stream = self.pubsub_management.read_stream(stream_id)
stream_route = stream.stream_route
publisher = StandaloneStreamPublisher(stream_id=stream_id,
stream_route=stream_route)
publisher.publish(msg)
def test_stream_crud(self):
stream_def_id = self.pubsub_management.create_stream_definition(
'test_definition', stream_type='stream')
topic_id = self.pubsub_management.create_topic(
name='test_topic', exchange_point='test_exchange')
self.exchange_cleanup.append('test_exchange')
topic2_id = self.pubsub_management.create_topic(
name='another_topic', exchange_point='outside')
stream_id, route = self.pubsub_management.create_stream(
name='test_stream',
topic_ids=[topic_id, topic2_id],
exchange_point='test_exchange',
stream_definition_id=stream_def_id)
topics, assocs = self.resource_registry.find_objects(
subject=stream_id, predicate=PRED.hasTopic, id_only=True)
self.assertEquals(topics, [topic_id])
defs, assocs = self.resource_registry.find_objects(
subject=stream_id,
predicate=PRED.hasStreamDefinition,
id_only=True)
self.assertTrue(len(defs))
stream = self.pubsub_management.read_stream(stream_id)
self.assertEquals(stream.name, 'test_stream')
self.pubsub_management.delete_stream(stream_id)
with self.assertRaises(NotFound):
self.pubsub_management.read_stream(stream_id)
defs, assocs = self.resource_registry.find_objects(
subject=stream_id,
predicate=PRED.hasStreamDefinition,
id_only=True)
self.assertFalse(len(defs))
topics, assocs = self.resource_registry.find_objects(
subject=stream_id, predicate=PRED.hasTopic, id_only=True)
self.assertFalse(len(topics))
self.pubsub_management.delete_topic(topic_id)
self.pubsub_management.delete_topic(topic2_id)
self.pubsub_management.delete_stream_definition(stream_def_id)
def test_subscription_crud(self):
stream_def_id = self.pubsub_management.create_stream_definition(
'test_definition', stream_type='stream')
stream_id, route = self.pubsub_management.create_stream(
name='test_stream',
exchange_point='test_exchange',
stream_definition_id=stream_def_id)
subscription_id = self.pubsub_management.create_subscription(
name='test subscription',
stream_ids=[stream_id],
exchange_name='test_queue')
self.exchange_cleanup.append('test_exchange')
subs, assocs = self.resource_registry.find_objects(
subject=subscription_id, predicate=PRED.hasStream, id_only=True)
self.assertEquals(subs, [stream_id])
res, _ = self.resource_registry.find_resources(restype=RT.ExchangeName,
name='test_queue',
id_only=True)
self.assertEquals(len(res), 1)
subs, assocs = self.resource_registry.find_subjects(
object=subscription_id,
predicate=PRED.hasSubscription,
id_only=True)
self.assertEquals(subs[0], res[0])
subscription = self.pubsub_management.read_subscription(
subscription_id)
self.assertEquals(subscription.exchange_name, 'test_queue')
self.pubsub_management.delete_subscription(subscription_id)
subs, assocs = self.resource_registry.find_objects(
subject=subscription_id, predicate=PRED.hasStream, id_only=True)
self.assertFalse(len(subs))
subs, assocs = self.resource_registry.find_subjects(
object=subscription_id,
predicate=PRED.hasSubscription,
id_only=True)
self.assertFalse(len(subs))
self.pubsub_management.delete_stream(stream_id)
self.pubsub_management.delete_stream_definition(stream_def_id)
def test_move_before_activate(self):
stream_id, route = self.pubsub_management.create_stream(
name='test_stream', exchange_point='test_xp')
#--------------------------------------------------------------------------------
# Test moving before activate
#--------------------------------------------------------------------------------
subscription_id = self.pubsub_management.create_subscription(
'first_queue', stream_ids=[stream_id])
xn_ids, _ = self.resource_registry.find_resources(
restype=RT.ExchangeName, name='first_queue', id_only=True)
subjects, _ = self.resource_registry.find_subjects(
object=subscription_id,
predicate=PRED.hasSubscription,
id_only=True)
self.assertEquals(xn_ids[0], subjects[0])
self.pubsub_management.move_subscription(subscription_id,
exchange_name='second_queue')
xn_ids, _ = self.resource_registry.find_resources(
restype=RT.ExchangeName, name='second_queue', id_only=True)
subjects, _ = self.resource_registry.find_subjects(
object=subscription_id,
predicate=PRED.hasSubscription,
id_only=True)
self.assertEquals(len(subjects), 1)
self.assertEquals(subjects[0], xn_ids[0])
self.pubsub_management.delete_subscription(subscription_id)
self.pubsub_management.delete_stream(stream_id)
def test_move_activated_subscription(self):
stream_id, route = self.pubsub_management.create_stream(
name='test_stream', exchange_point='test_xp')
#--------------------------------------------------------------------------------
# Test moving after activate
#--------------------------------------------------------------------------------
subscription_id = self.pubsub_management.create_subscription(
'first_queue', stream_ids=[stream_id])
self.pubsub_management.activate_subscription(subscription_id)
xn_ids, _ = self.resource_registry.find_resources(
restype=RT.ExchangeName, name='first_queue', id_only=True)
subjects, _ = self.resource_registry.find_subjects(
object=subscription_id,
predicate=PRED.hasSubscription,
id_only=True)
self.assertEquals(xn_ids[0], subjects[0])
self.verified = Event()
def verify(m, r, s):
self.assertEquals(m, 'verified')
self.verified.set()
subscriber = StandaloneStreamSubscriber('second_queue', verify)
subscriber.start()
self.pubsub_management.move_subscription(subscription_id,
exchange_name='second_queue')
xn_ids, _ = self.resource_registry.find_resources(
restype=RT.ExchangeName, name='second_queue', id_only=True)
subjects, _ = self.resource_registry.find_subjects(
object=subscription_id,
predicate=PRED.hasSubscription,
id_only=True)
self.assertEquals(len(subjects), 1)
self.assertEquals(subjects[0], xn_ids[0])
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish('verified')
self.assertTrue(self.verified.wait(2))
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
self.pubsub_management.delete_stream(stream_id)
def test_queue_cleanup(self):
stream_id, route = self.pubsub_management.create_stream(
'test_stream', 'xp1')
xn_objs, _ = self.resource_registry.find_resources(
restype=RT.ExchangeName, name='queue1')
for xn_obj in xn_objs:
xn = self.container.ex_manager.create_xn_queue(xn_obj.name)
xn.delete()
subscription_id = self.pubsub_management.create_subscription(
'queue1', stream_ids=[stream_id])
xn_ids, _ = self.resource_registry.find_resources(
restype=RT.ExchangeName, name='queue1')
self.assertEquals(len(xn_ids), 1)
self.pubsub_management.delete_subscription(subscription_id)
xn_ids, _ = self.resource_registry.find_resources(
restype=RT.ExchangeName, name='queue1')
self.assertEquals(len(xn_ids), 0)
def test_activation_and_deactivation(self):
stream_id, route = self.pubsub_management.create_stream(
'stream1', 'xp1')
subscription_id = self.pubsub_management.create_subscription(
'sub1', stream_ids=[stream_id])
self.check1 = Event()
def verifier(m, r, s):
self.check1.set()
subscriber = StandaloneStreamSubscriber('sub1', verifier)
subscriber.start()
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish('should not receive')
self.assertFalse(self.check1.wait(0.25))
self.pubsub_management.activate_subscription(subscription_id)
publisher.publish('should receive')
self.assertTrue(self.check1.wait(2))
self.check1.clear()
self.assertFalse(self.check1.is_set())
self.pubsub_management.deactivate_subscription(subscription_id)
publisher.publish('should not receive')
self.assertFalse(self.check1.wait(0.5))
self.pubsub_management.activate_subscription(subscription_id)
publisher.publish('should receive')
self.assertTrue(self.check1.wait(2))
subscriber.stop()
self.pubsub_management.deactivate_subscription(subscription_id)
self.pubsub_management.delete_subscription(subscription_id)
self.pubsub_management.delete_stream(stream_id)
def test_topic_crud(self):
topic_id = self.pubsub_management.create_topic(
name='test_topic', exchange_point='test_xp')
self.exchange_cleanup.append('test_xp')
topic = self.pubsub_management.read_topic(topic_id)
self.assertEquals(topic.name, 'test_topic')
self.assertEquals(topic.exchange_point, 'test_xp')
self.pubsub_management.delete_topic(topic_id)
with self.assertRaises(NotFound):
self.pubsub_management.read_topic(topic_id)
def test_full_pubsub(self):
self.sub1_sat = Event()
self.sub2_sat = Event()
def subscriber1(m, r, s):
self.sub1_sat.set()
def subscriber2(m, r, s):
self.sub2_sat.set()
sub1 = StandaloneStreamSubscriber('sub1', subscriber1)
self.queue_cleanup.append(sub1.xn.queue)
sub1.start()
sub2 = StandaloneStreamSubscriber('sub2', subscriber2)
self.queue_cleanup.append(sub2.xn.queue)
sub2.start()
log_topic = self.pubsub_management.create_topic(
'instrument_logs', exchange_point='instruments')
science_topic = self.pubsub_management.create_topic(
'science_data', exchange_point='instruments')
events_topic = self.pubsub_management.create_topic(
'notifications', exchange_point='events')
log_stream, route = self.pubsub_management.create_stream(
'instrument1-logs',
topic_ids=[log_topic],
exchange_point='instruments')
ctd_stream, route = self.pubsub_management.create_stream(
'instrument1-ctd',
topic_ids=[science_topic],
exchange_point='instruments')
event_stream, route = self.pubsub_management.create_stream(
'notifications', topic_ids=[events_topic], exchange_point='events')
raw_stream, route = self.pubsub_management.create_stream(
'temp', exchange_point='global.data')
self.exchange_cleanup.extend(['instruments', 'events', 'global.data'])
subscription1 = self.pubsub_management.create_subscription(
'subscription1',
stream_ids=[log_stream, event_stream],
exchange_name='sub1')
subscription2 = self.pubsub_management.create_subscription(
'subscription2',
exchange_points=['global.data'],
stream_ids=[ctd_stream],
exchange_name='sub2')
self.pubsub_management.activate_subscription(subscription1)
self.pubsub_management.activate_subscription(subscription2)
self.publish_on_stream(log_stream, 1)
self.assertTrue(self.sub1_sat.wait(4))
self.assertFalse(self.sub2_sat.is_set())
self.publish_on_stream(raw_stream, 1)
self.assertTrue(self.sub1_sat.wait(4))
sub1.stop()
sub2.stop()
def test_topic_craziness(self):
self.msg_queue = Queue()
def subscriber1(m, r, s):
self.msg_queue.put(m)
sub1 = StandaloneStreamSubscriber('sub1', subscriber1)
self.queue_cleanup.append(sub1.xn.queue)
sub1.start()
topic1 = self.pubsub_management.create_topic('topic1',
exchange_point='xp1')
topic2 = self.pubsub_management.create_topic('topic2',
exchange_point='xp1',
parent_topic_id=topic1)
topic3 = self.pubsub_management.create_topic('topic3',
exchange_point='xp1',
parent_topic_id=topic1)
topic4 = self.pubsub_management.create_topic('topic4',
exchange_point='xp1',
parent_topic_id=topic2)
topic5 = self.pubsub_management.create_topic('topic5',
exchange_point='xp1',
parent_topic_id=topic2)
topic6 = self.pubsub_management.create_topic('topic6',
exchange_point='xp1',
parent_topic_id=topic3)
topic7 = self.pubsub_management.create_topic('topic7',
exchange_point='xp1',
parent_topic_id=topic3)
# Tree 2
topic8 = self.pubsub_management.create_topic('topic8',
exchange_point='xp2')
topic9 = self.pubsub_management.create_topic('topic9',
exchange_point='xp2',
parent_topic_id=topic8)
topic10 = self.pubsub_management.create_topic('topic10',
exchange_point='xp2',
parent_topic_id=topic9)
topic11 = self.pubsub_management.create_topic('topic11',
exchange_point='xp2',
parent_topic_id=topic9)
topic12 = self.pubsub_management.create_topic('topic12',
exchange_point='xp2',
parent_topic_id=topic11)
topic13 = self.pubsub_management.create_topic('topic13',
exchange_point='xp2',
parent_topic_id=topic11)
self.exchange_cleanup.extend(['xp1', 'xp2'])
stream1_id, route = self.pubsub_management.create_stream(
'stream1',
topic_ids=[topic7, topic4, topic5],
exchange_point='xp1')
stream2_id, route = self.pubsub_management.create_stream(
'stream2', topic_ids=[topic8], exchange_point='xp2')
stream3_id, route = self.pubsub_management.create_stream(
'stream3', topic_ids=[topic10, topic13], exchange_point='xp2')
stream4_id, route = self.pubsub_management.create_stream(
'stream4', topic_ids=[topic9], exchange_point='xp2')
stream5_id, route = self.pubsub_management.create_stream(
'stream5', topic_ids=[topic11], exchange_point='xp2')
subscription1 = self.pubsub_management.create_subscription(
'sub1', topic_ids=[topic1])
subscription2 = self.pubsub_management.create_subscription(
'sub2', topic_ids=[topic8], exchange_name='sub1')
subscription3 = self.pubsub_management.create_subscription(
'sub3', topic_ids=[topic9], exchange_name='sub1')
subscription4 = self.pubsub_management.create_subscription(
'sub4',
topic_ids=[topic10, topic13, topic11],
exchange_name='sub1')
#--------------------------------------------------------------------------------
self.pubsub_management.activate_subscription(subscription1)
self.publish_on_stream(stream1_id, 1)
self.assertEquals(self.msg_queue.get(timeout=10), 1)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.1)
self.pubsub_management.deactivate_subscription(subscription1)
self.pubsub_management.delete_subscription(subscription1)
#--------------------------------------------------------------------------------
self.pubsub_management.activate_subscription(subscription2)
self.publish_on_stream(stream2_id, 2)
self.assertEquals(self.msg_queue.get(timeout=10), 2)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.1)
self.pubsub_management.deactivate_subscription(subscription2)
self.pubsub_management.delete_subscription(subscription2)
#--------------------------------------------------------------------------------
self.pubsub_management.activate_subscription(subscription3)
self.publish_on_stream(stream2_id, 3)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.3)
self.publish_on_stream(stream3_id, 4)
self.assertEquals(self.msg_queue.get(timeout=10), 4)
self.pubsub_management.deactivate_subscription(subscription3)
self.pubsub_management.delete_subscription(subscription3)
#--------------------------------------------------------------------------------
self.pubsub_management.activate_subscription(subscription4)
self.publish_on_stream(stream4_id, 5)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.3)
self.publish_on_stream(stream5_id, 6)
self.assertEquals(self.msg_queue.get(timeout=10), 6)
with self.assertRaises(Empty):
self.msg_queue.get(timeout=0.3)
self.pubsub_management.deactivate_subscription(subscription4)
self.pubsub_management.delete_subscription(subscription4)
#--------------------------------------------------------------------------------
sub1.stop()
self.pubsub_management.delete_topic(topic13)
self.pubsub_management.delete_topic(topic12)
self.pubsub_management.delete_topic(topic11)
self.pubsub_management.delete_topic(topic10)
self.pubsub_management.delete_topic(topic9)
self.pubsub_management.delete_topic(topic8)
self.pubsub_management.delete_topic(topic7)
self.pubsub_management.delete_topic(topic6)
self.pubsub_management.delete_topic(topic5)
self.pubsub_management.delete_topic(topic4)
self.pubsub_management.delete_topic(topic3)
self.pubsub_management.delete_topic(topic2)
self.pubsub_management.delete_topic(topic1)
self.pubsub_management.delete_stream(stream1_id)
self.pubsub_management.delete_stream(stream2_id)
self.pubsub_management.delete_stream(stream3_id)
self.pubsub_management.delete_stream(stream4_id)
self.pubsub_management.delete_stream(stream5_id)
def _get_pdict(self, filter_values):
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='TIME',
parameter_context=t_ctxt.dump(),
parameter_type='quantity<int64>',
unit_of_measure=t_ctxt.uom)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='LAT',
parameter_context=lat_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lat_ctxt.uom)
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='LON',
parameter_context=lon_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lon_ctxt.uom)
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='TEMPWAT_L0',
parameter_context=temp_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=temp_ctxt.uom)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='CONDWAT_L0',
parameter_context=cond_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=cond_ctxt.uom)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='PRESWAT_L0',
parameter_context=press_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=press_ctxt.uom)
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr = NumexprFunction('TEMPWAT_L1',
tl1_func, ['T'],
param_map=tl1_pmap)
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name=tempL1_ctxt.name,
parameter_context=tempL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=tempL1_ctxt.uom)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
cl1_pmap = {'C': 'CONDWAT_L0'}
expr = NumexprFunction('CONDWAT_L1',
cl1_func, ['C'],
param_map=cl1_pmap)
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name=condL1_ctxt.name,
parameter_context=condL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=condL1_ctxt.uom)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr = NumexprFunction('PRESWAT_L1',
pl1_func, ['P', 'p_range'],
param_map=pl1_pmap)
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name=presL1_ctxt.name,
parameter_context=presL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=presL1_ctxt.uom)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
sal_kwargmap = None
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist,
sal_kwargmap, sal_pmap)
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name=sal_ctxt.name,
parameter_context=sal_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=sal_ctxt.uom)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name=dens_ctxt.name,
parameter_context=dens_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=dens_ctxt.uom)
ids = [
t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id,
press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id,
sal_ctxt_id, dens_ctxt_id
]
contexts = [
t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt,
tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt
]
context_ids = [
ids[i] for i, ctxt in enumerate(contexts)
if ctxt.name in filter_values
]
pdict_name = '_'.join(
[ctxt.name for ctxt in contexts if ctxt.name in filter_values])
try:
self.pdicts[pdict_name]
return self.pdicts[pdict_name]
except KeyError:
pdict_id = self.dataset_management.create_parameter_dictionary(
pdict_name,
parameter_context_ids=context_ids,
temporal_context='time')
self.pdicts[pdict_name] = pdict_id
return pdict_id
class TestInstrumentAgent(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def customCleanUp(self):
log.info('CUSTOM CLEAN UP ******************************************************************************')
def setUp(self):
"""
Setup the test environment to exersice use of instrumet agent, including:
* define driver_config parameters.
* create container with required services and container client.
* create publication stream ids for each driver data stream.
* create stream_config parameters.
* create and activate subscriptions for agent data streams.
* spawn instrument agent process and create agent client.
* add cleanup functions to cause subscribers to get stopped.
"""
# params = { ('CTD', 'TA2'): -1.9434316e-05,
# ('CTD', 'PTCA1'): 1.3206866,
# ('CTD', 'TCALDATE'): [8, 11, 2006] }
# for tup in params:
# print tup
self.addCleanup(self.customCleanUp)
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.services.mi.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
'''
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.services.mi.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': '137.110.112.119',
'device_port': 4001,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
'''
# Start container.
self._start_container()
# Establish endpoint with container (used in tests below)
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
# Create a pubsub client to create streams.
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
container=self.container)
self.subs = []
# Create streams for each stream named in driver.
self.stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = self._pubsub_client.create_stream_definition(
container=stream_def)
stream_id = self._pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self.stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = self._pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name)
self._pubsub_client.activate_subscription(sub_id)
self.subs.append(sub)
# Add cleanup function to stop subscribers.
def stop_subscriber(sub_list):
for sub in sub_list:
sub.stop()
self.addCleanup(stop_subscriber, self.subs)
# Create agent config.
self.agent_resource_id = '123xyz'
self.agent_config = {
'driver_config' : self.driver_config,
'stream_config' : self.stream_config,
'agent' : {'resource_id': self.agent_resource_id}
}
# Launch an instrument agent process.
self._ia_name = 'agent007'
self._ia_mod = 'ion.services.mi.instrument_agent'
self._ia_class = 'InstrumentAgent'
self._ia_pid = self._container_client.spawn_process(name=self._ia_name,
module=self._ia_mod, cls=self._ia_class,
config=self.agent_config)
log.info('got pid=%s', str(self._ia_pid))
self._ia_client = None
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(self.agent_resource_id, process=FakeProcess())
log.info('got ia client %s', str(self._ia_client))
def test_initialize(self):
"""
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
log.info(reply)
time.sleep(2)
caps = gw_agent_get_capabilities(self.agent_resource_id)
log.info('Capabilities: %s',str(caps))
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
log.info(reply)
def test_direct_access(self):
"""
Test agent direct_access command. This causes creation of
driver process and transition to direct access.
"""
print("test initing")
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_active")
cmd = AgentCommand(command='go_active')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test run")
cmd = AgentCommand(command='run')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_da")
cmd = AgentCommand(command='go_direct_access')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
print("reply=" + str(reply))
time.sleep(2)
print("test go_ob")
cmd = AgentCommand(command='go_observatory')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_inactive")
cmd = AgentCommand(command='go_inactive')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test reset")
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
def test_go_active(self):
"""
Test agent go_active command. This causes a driver process to
launch a connection broker, connect to device hardware, determine
entry state of driver and initialize driver parameters.
"""
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='go_inactive')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
class CtdTransformsIntTest(IonIntegrationTestCase):
def setUp(self):
super(CtdTransformsIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.queue_cleanup = []
self.exchange_cleanup = []
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.exchange_name = 'ctd_L0_all_queue'
self.exchange_point = 'test_exchange'
self.i = 0
def tearDown(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
for exchange in self.exchange_cleanup:
xp = self.container.ex_manager.create_xp(exchange)
xp.delete()
def test_ctd_L0_all(self):
'''
Test that packets are processed by the ctd_L0_all transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='ctd_L0_all', description='For testing ctd_L0_all')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.ctd_L0_all'
process_definition.executable['class'] = 'ctd_L0_all'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'ctd_all_stream_def', parameter_dictionary_id=pdict_id)
cond_stream_id, _ = self.pubsub.create_stream(
'test_cond',
exchange_point='science_data',
stream_definition_id=stream_def_id)
pres_stream_id, _ = self.pubsub.create_stream(
'test_pres',
exchange_point='science_data',
stream_definition_id=stream_def_id)
temp_stream_id, _ = self.pubsub.create_stream(
'test_temp',
exchange_point='science_data',
stream_definition_id=stream_def_id)
config.process.publish_streams.conductivity = cond_stream_id
config.process.publish_streams.pressure = pres_stream_id
config.process.publish_streams.temperature = temp_stream_id
# Schedule the process
pid = self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscribers that will receive the conductivity, temperature and pressure granules from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber1(m, r, s):
ar_cond.set(m)
sub_cond = StandaloneStreamSubscriber('sub_cond', subscriber1)
self.addCleanup(sub_cond.stop)
ar_temp = gevent.event.AsyncResult()
def subscriber2(m, r, s):
ar_temp.set(m)
sub_temp = StandaloneStreamSubscriber('sub_temp', subscriber2)
self.addCleanup(sub_temp.stop)
ar_pres = gevent.event.AsyncResult()
def subscriber3(m, r, s):
ar_pres.set(m)
sub_pres = StandaloneStreamSubscriber('sub_pres', subscriber3)
self.addCleanup(sub_pres.stop)
sub_cond_id = self.pubsub.create_subscription(
'subscription_cond',
stream_ids=[cond_stream_id],
exchange_name='sub_cond')
sub_temp_id = self.pubsub.create_subscription(
'subscription_temp',
stream_ids=[temp_stream_id],
exchange_name='sub_temp')
sub_pres_id = self.pubsub.create_subscription(
'subscription_pres',
stream_ids=[pres_stream_id],
exchange_name='sub_pres')
self.pubsub.activate_subscription(sub_cond_id)
self.pubsub.activate_subscription(sub_temp_id)
self.pubsub.activate_subscription(sub_pres_id)
self.queue_cleanup.append(sub_cond.xn.queue)
self.queue_cleanup.append(sub_temp.xn.queue)
self.queue_cleanup.append(sub_pres.xn.queue)
sub_cond.start()
sub_temp.start()
sub_pres.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
publish_granule = self._get_new_ctd_packet(
stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result_cond = ar_cond.get(timeout=10)
result_temp = ar_temp.get(timeout=10)
result_pres = ar_pres.get(timeout=10)
out_dict = {}
out_dict['c'] = RecordDictionaryTool.load_from_granule(
result_cond)['conductivity']
out_dict['t'] = RecordDictionaryTool.load_from_granule(
result_temp)['temp']
out_dict['p'] = RecordDictionaryTool.load_from_granule(
result_pres)['pressure']
# Check that the transform algorithm was successfully executed
self.check_granule_splitting(publish_granule, out_dict)
def test_ctd_L1_conductivity(self):
'''
Test that packets are processed by the ctd_L1_conductivity transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='CTDL1ConductivityTransform',
description='For testing CTDL1ConductivityTransform')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.ctd_L1_conductivity'
process_definition.executable['class'] = 'CTDL1ConductivityTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'cond_stream_def', parameter_dictionary_id=pdict_id)
cond_stream_id, _ = self.pubsub.create_stream(
'test_conductivity',
exchange_point='science_data',
stream_definition_id=stream_def_id)
config.process.publish_streams.conductivity = cond_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscribers that will receive the conductivity, temperature and pressure granules from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber1(m, r, s):
ar_cond.set(m)
sub_cond = StandaloneStreamSubscriber('sub_cond', subscriber1)
self.addCleanup(sub_cond.stop)
sub_cond_id = self.pubsub.create_subscription(
'subscription_cond',
stream_ids=[cond_stream_id],
exchange_name='sub_cond')
self.pubsub.activate_subscription(sub_cond_id)
self.queue_cleanup.append(sub_cond.xn.queue)
sub_cond.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
publish_granule = self._get_new_ctd_packet(
stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result_cond = ar_cond.get(timeout=10)
self.assertTrue(isinstance(result_cond, Granule))
rdt = RecordDictionaryTool.load_from_granule(result_cond)
self.assertTrue(rdt.__contains__('conductivity'))
self.check_cond_algorithm_execution(publish_granule, result_cond)
def check_cond_algorithm_execution(self, publish_granule,
granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(
granule_from_transform)
output_data = output_rdt_transform['conductivity']
input_data = input_rdt_to_transform['conductivity']
self.assertTrue(
numpy.array_equal(((input_data / 100000.0) - 0.5), output_data))
def check_pres_algorithm_execution(self, publish_granule,
granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(
granule_from_transform)
output_data = output_rdt_transform['pressure']
input_data = input_rdt_to_transform['pressure']
self.assertTrue(
numpy.array_equal((input_data / 100.0) + 0.5, output_data))
def check_temp_algorithm_execution(self, publish_granule,
granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(
granule_from_transform)
output_data = output_rdt_transform['temp']
input_data = input_rdt_to_transform['temp']
self.assertTrue(
numpy.array_equal(((input_data / 10000.0) - 10), output_data))
def check_density_algorithm_execution(self, publish_granule,
granule_from_transform):
#------------------------------------------------------------------
# Calculate the correct density from the input granule data
#------------------------------------------------------------------
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(
granule_from_transform)
conductivity = input_rdt_to_transform['conductivity']
pressure = input_rdt_to_transform['pressure']
temperature = input_rdt_to_transform['temp']
longitude = input_rdt_to_transform['lon']
latitude = input_rdt_to_transform['lat']
sp = SP_from_cndr(r=conductivity / cte.C3515,
t=temperature,
p=pressure)
sa = SA_from_SP(sp, pressure, longitude, latitude)
dens_value = rho(sa, temperature, pressure)
out_density = output_rdt_transform['density']
log.debug("density output from the transform: %s", out_density)
log.debug("values of density expected from the transform: %s",
dens_value)
numpy.testing.assert_array_almost_equal(out_density,
dens_value,
decimal=3)
# #-----------------------------------------------------------------------------
# # Check that the output data from the transform has the correct density values
# #-----------------------------------------------------------------------------
# for item in zip(out_density.tolist(), dens_value.tolist()):
# if isinstance(item[0], int) or isinstance(item[0], float):
# self.assertEquals(item[0], item[1])
def check_salinity_algorithm_execution(self, publish_granule,
granule_from_transform):
#------------------------------------------------------------------
# Calculate the correct density from the input granule data
#------------------------------------------------------------------
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(
granule_from_transform)
conductivity = input_rdt_to_transform['conductivity']
pressure = input_rdt_to_transform['pressure']
temperature = input_rdt_to_transform['temp']
sal_value = SP_from_cndr(r=conductivity / cte.C3515,
t=temperature,
p=pressure)
out_salinity = output_rdt_transform['salinity']
#-----------------------------------------------------------------------------
# Check that the output data from the transform has the correct density values
#-----------------------------------------------------------------------------
self.assertTrue(numpy.array_equal(sal_value, out_salinity))
def check_granule_splitting(self, publish_granule, out_dict):
'''
This checks that the ctd_L0_all transform is able to split out one of the
granules from the whole granule
fed into the transform
'''
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
in_cond = input_rdt_to_transform['conductivity']
in_pressure = input_rdt_to_transform['pressure']
in_temp = input_rdt_to_transform['temp']
out_cond = out_dict['c']
out_pres = out_dict['p']
out_temp = out_dict['t']
self.assertTrue(numpy.array_equal(in_cond, out_cond))
self.assertTrue(numpy.array_equal(in_pressure, out_pres))
self.assertTrue(numpy.array_equal(in_temp, out_temp))
def test_ctd_L1_pressure(self):
'''
Test that packets are processed by the ctd_L1_pressure transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='CTDL1PressureTransform',
description='For testing CTDL1PressureTransform')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.ctd_L1_pressure'
process_definition.executable['class'] = 'CTDL1PressureTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'pres_stream_def', parameter_dictionary_id=pdict_id)
pres_stream_id, _ = self.pubsub.create_stream(
'test_pressure',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.pressure = pres_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscribers that will receive the pressure granules from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_pres = gevent.event.AsyncResult()
def subscriber3(m, r, s):
ar_pres.set(m)
sub_pres = StandaloneStreamSubscriber('sub_pres', subscriber3)
self.addCleanup(sub_pres.stop)
sub_pres_id = self.pubsub.create_subscription(
'subscription_pres',
stream_ids=[pres_stream_id],
exchange_name='sub_pres')
self.pubsub.activate_subscription(sub_pres_id)
self.queue_cleanup.append(sub_pres.xn.queue)
sub_pres.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
publish_granule = self._get_new_ctd_packet(
stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_pres.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('pressure'))
self.check_pres_algorithm_execution(publish_granule, result)
def test_ctd_L1_temperature(self):
'''
Test that packets are processed by the ctd_L1_temperature transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='CTDL1TemperatureTransform',
description='For testing CTDL1TemperatureTransform')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.ctd_L1_temperature'
process_definition.executable['class'] = 'CTDL1TemperatureTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'temp_stream_def', parameter_dictionary_id=pdict_id)
temp_stream_id, _ = self.pubsub.create_stream(
'test_temperature',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.temperature = temp_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscriber that will receive the temperature granule from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_temp = gevent.event.AsyncResult()
def subscriber2(m, r, s):
ar_temp.set(m)
sub_temp = StandaloneStreamSubscriber('sub_temp', subscriber2)
self.addCleanup(sub_temp.stop)
sub_temp_id = self.pubsub.create_subscription(
'subscription_temp',
stream_ids=[temp_stream_id],
exchange_name='sub_temp')
self.pubsub.activate_subscription(sub_temp_id)
self.queue_cleanup.append(sub_temp.xn.queue)
sub_temp.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
publish_granule = self._get_new_ctd_packet(
stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_temp.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('temp'))
self.check_temp_algorithm_execution(publish_granule, result)
def test_ctd_L2_density(self):
'''
Test that packets are processed by the ctd_L1_density transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='DensityTransform',
description='For testing DensityTransform')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.ctd_L2_density'
process_definition.executable['class'] = 'DensityTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
config.process.interval = 1.0
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'dens_stream_def', parameter_dictionary_id=pdict_id)
dens_stream_id, _ = self.pubsub.create_stream(
'test_density',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.density = dens_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Create a subscriber that will receive the density granule from the ctd transform
#---------------------------------------------------------------------------------------------
ar_dens = gevent.event.AsyncResult()
def subscriber3(m, r, s):
ar_dens.set(m)
sub_dens = StandaloneStreamSubscriber('sub_dens', subscriber3)
self.addCleanup(sub_dens.stop)
sub_dens_id = self.pubsub.create_subscription(
'subscription_dens',
stream_ids=[dens_stream_id],
exchange_name='sub_dens')
self.pubsub.activate_subscription(sub_dens_id)
self.queue_cleanup.append(sub_dens.xn.queue)
sub_dens.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
publish_granule = self._get_new_ctd_packet(
stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_dens.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('density'))
self.check_density_algorithm_execution(publish_granule, result)
def test_ctd_L2_salinity(self):
'''
Test that packets are processed by the ctd_L1_salinity transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='SalinityTransform',
description='For testing SalinityTransform')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.ctd_L2_salinity'
process_definition.executable['class'] = 'SalinityTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'sal_stream_def', parameter_dictionary_id=pdict_id)
sal_stream_id, _ = self.pubsub.create_stream(
'test_salinity',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.salinity = sal_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Create a subscriber that will receive the salinity granule from the ctd transform
#---------------------------------------------------------------------------------------------
ar_sal = gevent.event.AsyncResult()
def subscriber3(m, r, s):
ar_sal.set(m)
sub_sal = StandaloneStreamSubscriber('sub_sal', subscriber3)
self.addCleanup(sub_sal.stop)
sub_sal_id = self.pubsub.create_subscription(
'subscription_sal',
stream_ids=[sal_stream_id],
exchange_name='sub_sal')
self.pubsub.activate_subscription(sub_sal_id)
self.queue_cleanup.append(sub_sal.xn.queue)
sub_sal.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
publish_granule = self._get_new_ctd_packet(
stream_definition_id=stream_def_id, length=5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_sal.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('salinity'))
self.check_salinity_algorithm_execution(publish_granule, result)
def _get_new_ctd_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
rdt['time'] = numpy.arange(self.i, self.i + length)
for field in rdt:
if isinstance(
rdt._pdict.get_context(field).param_type, QuantityType):
rdt[field] = numpy.array(
[random.uniform(0.0, 75.0) for i in xrange(length)])
g = rdt.to_granule()
self.i += length
return g
def test_presf_L0_splitter(self):
'''
Test that packets are processed by the ctd_L1_pressure transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='Presf L0 Splitter',
description='For testing Presf L0 Splitter')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.presf_L0_splitter'
process_definition.executable['class'] = 'PresfL0Splitter'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'pres_stream_def', parameter_dictionary_id=pdict_id)
pres_stream_id, _ = self.pubsub.create_stream(
'test_pressure',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.absolute_pressure = pres_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
# #---------------------------------------------------------------------------------------------
# # Create subscribers that will receive the pressure granules from
# # the ctd transform
# #---------------------------------------------------------------------------------------------
#
# ar_pres = gevent.event.AsyncResult()
# def subscriber3(m,r,s):
# ar_pres.set(m)
# sub_pres = StandaloneStreamSubscriber('sub_pres', subscriber3)
# self.addCleanup(sub_pres.stop)
#
# sub_pres_id = self.pubsub.create_subscription('subscription_pres',
# stream_ids=[pres_stream_id],
# exchange_name='sub_pres')
#
# self.pubsub.activate_subscription(sub_pres_id)
#
# self.queue_cleanup.append(sub_pres.xn.queue)
#
# sub_pres.start()
#
# #------------------------------------------------------------------------------------------------------
# # Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
# #------------------------------------------------------------------------------------------------------
#
# # Do all the routing stuff for the publishing
# routing_key = 'stream_id.stream'
# stream_route = StreamRoute(self.exchange_point, routing_key)
#
# xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
# xp = self.container.ex_manager.create_xp(self.exchange_point)
# xn.bind('stream_id.stream', xp)
#
# pub = StandaloneStreamPublisher('stream_id', stream_route)
#
# # Build a packet that can be published
# publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
#
# # Publish the packet
# pub.publish(publish_granule)
#
# #------------------------------------------------------------------------------------------------------
# # Make assertions about whether the ctd transform executed its algorithm and published the correct
# # granules
# #------------------------------------------------------------------------------------------------------
#
# # Get the granule that is published by the ctd transform post processing
# result = ar_pres.get(timeout=10)
# self.assertTrue(isinstance(result, Granule))
#
# rdt = RecordDictionaryTool.load_from_granule(result)
# self.assertTrue(rdt.__contains__('pressure'))
#
# self.check_pres_algorithm_execution(publish_granule, result)
#
def test_presf_L1(self):
'''
Test that packets are processed by the ctd_L1_pressure transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='PresfL1Transform',
description='For testing PresfL1Transform')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.ctd.presf_L1'
process_definition.executable['class'] = 'PresfL1Transform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'pres_stream_def', parameter_dictionary_id=pdict_id)
pres_stream_id, _ = self.pubsub.create_stream(
'test_pressure',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.seafloor_pressure = pres_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=ctd_transform_proc_def_id,
configuration=config)
def run_even_odd_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition,
name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class': 'TransformExample'
}
basic_transform_definition_id = procd_cli.create_process_definition(
process_definition=process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition,
name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class': 'TransformEvenOdd'
}
evenodd_transform_definition_id = procd_cli.create_process_definition(
process_definition=process_definition)
#-------------------------------
# Streams
#-------------------------------
input_stream_id = pubsub_cli.create_stream(name='input_stream',
original=True)
even_stream_id = pubsub_cli.create_stream(name='even_stream',
original=True)
odd_stream_id = pubsub_cli.create_stream(name='odd_stream',
original=True)
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[input_stream_id])
input_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids=[even_stream_id])
even_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids=[odd_stream_id])
odd_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(
name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={
'even': even_stream_id,
'odd': odd_stream_id
},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(
name='even_transform',
in_subscription_id=even_subscription_id,
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(
name='odd_transform',
in_subscription_id=odd_subscription_id,
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Spawn the Streaming Producer
#-------------------------------
id_p = self.container.spawn_process(
'myproducer', 'ion.processes.data.transforms.transform_example',
'TransformExampleProducer', {
'process': {
'type': 'stream_process',
'publish_streams': {
'out_stream': input_stream_id
}
},
'stream_producer': {
'interval': 4000
}
})
self.container.proc_manager.procs[id_p].start()
class TestStreamIngestionWorker(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.time_dom, self.spatial_dom = time_series_domain()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
self.stream_id, self.route_id = self.pubsub_client.create_stream(
name='parsed_stream',
stream_definition_id=self.stream_def_id,
exchange_point='science_data')
self.addCleanup(self.pubsub_client.delete_stream, self.stream_id)
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name='parsed_subscription')
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
self.publisher = StandaloneStreamPublisher(self.stream_id,
self.route_id)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_stream_ingestion_worker(self):
self.start_ingestion_worker()
context_ids, time_ctxt = self._create_param_contexts()
pdict_id = self.dataset_management_client.create_parameter_dictionary(
name='stream_ingestion_pdict',
parameter_context_ids=context_ids,
temporal_context='ingestion_timestamp')
self.addCleanup(
self.dataset_management_client.delete_parameter_dictionary,
pdict_id)
dataset_id = self.dataset_management_client.create_dataset(
name='fake_dataset',
description='fake_dataset',
stream_id=self.stream_id,
spatial_domain=self.spatial_dom.dump(),
temporal_domain=self.time_dom.dump(),
parameter_dictionary_id=pdict_id)
self.addCleanup(self.dataset_management_client.delete_dataset,
dataset_id)
self.cov = self._create_coverage(dataset_id=dataset_id,
parameter_dict_id=pdict_id,
time_dom=self.time_dom,
spatial_dom=self.spatial_dom)
self.addCleanup(self.cov.close)
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['conductivity'] = 1
rdt['pressure'] = 2
rdt['salinity'] = 3
self.start_listener(dataset_id)
self.publisher.publish(rdt.to_granule())
self.data_modified = Event()
self.data_modified.wait(30)
cov = self.get_coverage(dataset_id)
self.assertIsNotNone(cov.get_parameter_values('raw'))
deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
granule = retrieve_stream(dataset_id)
rdt_complex = RecordDictionaryTool.load_from_granule(granule)
rdt_complex['raw'] = [
deserializer.deserialize(i) for i in rdt_complex['raw']
]
for gran in rdt_complex['raw']:
rdt_new = RecordDictionaryTool.load_from_granule(gran)
self.assertIn(1, rdt_new['conductivity'])
self.assertIn(2, rdt_new['pressure'])
self.assertIn(3, rdt_new['salinity'])
cov.close()
def start_ingestion_worker(self):
config = DotDict()
config.process.queue_name = 'parsed_subscription'
self.container.spawn_process(
name='stream_ingestion_worker',
module='ion.processes.data.ingestion.stream_ingestion_worker',
cls='StreamIngestionWorker',
config=config)
def start_listener(self, dataset_id):
def cb(*args, **kwargs):
self.data_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id)
es.start()
self.addCleanup(es.stop)
def _create_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext(
'ingestion_timestamp',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management_client.create_parameter_context(
name='ingestion_timestamp', parameter_context=t_ctxt.dump())
context_ids.append(t_ctxt_id)
raw_ctxt = ParameterContext('raw', param_type=ArrayType())
raw_ctxt.uom = ''
context_ids.append(
self.dataset_management_client.create_parameter_context(
name='raw', parameter_context=raw_ctxt.dump()))
return context_ids, t_ctxt_id
def _create_coverage(self, dataset_id, parameter_dict_id, time_dom,
spatial_dom):
pd = self.dataset_management_client.read_parameter_dictionary(
parameter_dict_id)
pdict = ParameterDictionary.load(pd)
sdom = GridDomain.load(spatial_dom.dump())
tdom = GridDomain.load(time_dom.dump())
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
scov = SimplexCoverage(file_root,
dataset_id,
dataset_id,
parameter_dictionary=pdict,
temporal_domain=tdom,
spatial_domain=sdom)
return scov
def get_coverage(self, dataset_id, mode='w'):
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
coverage = AbstractCoverage.load(file_root, dataset_id, mode=mode)
return coverage
class TestInstrumentAgent(IonIntegrationTestCase):
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def customCleanUp(self):
log.info(
'CUSTOM CLEAN UP ******************************************************************************'
)
def setUp(self):
"""
Setup the test environment to exersice use of instrumet agent, including:
* define driver_config parameters.
* create container with required services and container client.
* create publication stream ids for each driver data stream.
* create stream_config parameters.
* create and activate subscriptions for agent data streams.
* spawn instrument agent process and create agent client.
* add cleanup functions to cause subscribers to get stopped.
"""
# params = { ('CTD', 'TA2'): -1.9434316e-05,
# ('CTD', 'PTCA1'): 1.3206866,
# ('CTD', 'TCALDATE'): [8, 11, 2006] }
# for tup in params:
# print tup
self.addCleanup(self.customCleanUp)
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method': 'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
'''
self.driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': '137.110.112.119',
'device_port': 4001,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
'''
# Start container.
self._start_container()
# Establish endpoint with container (used in tests below)
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2dm.yml')
# Create a pubsub client to create streams.
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(
process=self.container, node=self.container.node)
self.subs = []
# Create streams for each stream named in driver.
self.stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = self._pubsub_client.create_stream_definition(
container=stream_def)
stream_id = self._pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
self.stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(
exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = self._pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name)
self._pubsub_client.activate_subscription(sub_id)
self.subs.append(sub)
# Add cleanup function to stop subscribers.
def stop_subscriber(sub_list):
for sub in sub_list:
sub.stop()
self.addCleanup(stop_subscriber, self.subs)
# Create agent config.
self.agent_resource_id = '123xyz'
self.agent_config = {
'driver_config': self.driver_config,
'stream_config': self.stream_config,
'agent': {
'resource_id': self.agent_resource_id
}
}
# Launch an instrument agent process.
self._ia_name = 'agent007'
self._ia_mod = 'ion.agents.instrument.instrument_agent'
self._ia_class = 'InstrumentAgent'
self._ia_pid = self._container_client.spawn_process(
name=self._ia_name,
module=self._ia_mod,
cls=self._ia_class,
config=self.agent_config)
log.info('got pid=%s', str(self._ia_pid))
self._ia_client = None
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(self.agent_resource_id,
process=FakeProcess())
log.info('got ia client %s', str(self._ia_client))
def test_initialize(self):
"""
Test agent initialize command. This causes creation of
driver process and transition to inactive.
"""
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
log.info(reply)
time.sleep(2)
caps = gw_agent_get_capabilities(self.agent_resource_id)
log.info('Capabilities: %s', str(caps))
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
log.info(reply)
def test_direct_access(self):
"""
Test agent direct_access command. This causes creation of
driver process and transition to direct access.
"""
print("test initing")
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_active")
cmd = AgentCommand(command='go_active')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test run")
cmd = AgentCommand(command='run')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_da")
cmd = AgentCommand(command='go_direct_access')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
print("reply=" + str(reply))
time.sleep(2)
print("test go_ob")
cmd = AgentCommand(command='go_observatory')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test go_inactive")
cmd = AgentCommand(command='go_inactive')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
print("test reset")
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
def test_go_active(self):
"""
Test agent go_active command. This causes a driver process to
launch a connection broker, connect to device hardware, determine
entry state of driver and initialize driver parameters.
"""
cmd = AgentCommand(command='initialize')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='go_inactive')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
cmd = AgentCommand(command='reset')
reply = gw_agent_execute_agent(self.agent_resource_id, cmd)
time.sleep(2)
class EventManagementIntTest(IonIntegrationTestCase):
def setUp(self):
super(EventManagementIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.event_management = EventManagementServiceClient()
self.rrc = ResourceRegistryServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub = PubsubManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.queue_cleanup = []
self.exchange_cleanup = []
def tearDown(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
for exchange in self.exchange_cleanup:
xp = self.container.ex_manager.create_xp(exchange)
xp.delete()
def test_create_read_update_delete_event_type(self):
"""
Test that the CRUD method for event types work correctly
"""
event_type = EventType(name="an event type")
event_type.origin = 'instrument_1'
# create
event_type_id = self.event_management.create_event_type(event_type)
self.assertIsNotNone(event_type_id)
# read
read_event_type = self.event_management.read_event_type(event_type_id)
self.assertEquals(read_event_type.name, event_type.name)
self.assertEquals(read_event_type.origin, event_type.origin)
#update
read_event_type.origin = 'instrument_2'
read_event_type.producer = 'producer'
self.event_management.update_event_type(read_event_type)
updated_event_type = self.event_management.read_event_type(event_type_id)
self.assertEquals(updated_event_type.origin, 'instrument_2')
self.assertEquals(updated_event_type.producer, 'producer')
# delete
self.event_management.delete_event_type(event_type_id)
with self.assertRaises(NotFound):
self.event_management.read_event_type(event_type_id)
def test_create_read_update_delete_event_process_definition(self):
"""
Test that the CRUD methods for the event process definitions work correctly
"""
# Create
module = 'ion.processes.data.transforms.event_alert_transform'
class_name = 'EventAlertTransform'
procdef_id = self.event_management.create_event_process_definition(version='ver_1',
module=module,
class_name=class_name,
uri='http://hare.com',
arguments=['arg1', 'arg2'],
event_types=['ExampleDetectableEvent', 'type_2'],
sub_types=['sub_type_1'])
# Read
read_process_def = self.event_management.read_event_process_definition(procdef_id)
self.assertEquals(read_process_def.executable['module'], module)
self.assertEquals(read_process_def.executable['class'], class_name)
# Update
self.event_management.update_event_process_definition(event_process_definition_id=procdef_id,
class_name='StreamAlertTransform',
arguments=['arg3', 'arg4'],
event_types=['event_type_new'])
updated_event_process_def = self.event_management.read_event_process_definition(procdef_id)
self.assertEquals(updated_event_process_def.executable['class'], 'StreamAlertTransform')
self.assertEquals(updated_event_process_def.arguments, ['arg3', 'arg4'])
definition = updated_event_process_def.definition
self.assertEquals(updated_event_process_def.definition.event_types, ['event_type_new'])
# Delete
self.event_management.delete_event_process_definition(procdef_id)
with self.assertRaises(NotFound):
self.event_management.read_event_process_definition(procdef_id)
def test_event_in_stream_out_transform(self):
"""
Test the event-in/stream-out transform
"""
stream_id, _ = self.pubsub.create_stream('test_stream', exchange_point='science_data')
self.exchange_cleanup.append('science_data')
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='EventToStreamTransform',
description='For testing an event-in/stream-out transform')
process_definition.executable['module']= 'ion.processes.data.transforms.event_in_stream_out_transform'
process_definition.executable['class'] = 'EventToStreamTransform'
proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = 'test_queue'
config.process.exchange_point = 'science_data'
config.process.publish_streams.output = stream_id
config.process.event_type = 'ExampleDetectableEvent'
config.process.variables = ['voltage', 'temperature' ]
# Schedule the process
pid = self.process_dispatcher.schedule_process(process_definition_id=proc_def_id, configuration=config)
self.addCleanup(self.process_dispatcher.cancel_process,pid)
#---------------------------------------------------------------------------------------------
# Create a subscriber for testing
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber_callback(m, r, s):
ar_cond.set(m)
sub = StandaloneStreamSubscriber('sub', subscriber_callback)
self.addCleanup(sub.stop)
sub_id = self.pubsub.create_subscription('subscription_cond',
stream_ids=[stream_id],
exchange_name='sub')
self.pubsub.activate_subscription(sub_id)
self.queue_cleanup.append(sub.xn.queue)
sub.start()
gevent.sleep(4)
#---------------------------------------------------------------------------------------------
# Publish an event. The transform has been configured to receive this event
#---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ExampleDetectableEvent")
event_publisher.publish_event(origin = 'fake_origin', voltage = '5', temperature = '273')
# Assert that the transform processed the event and published data on the output stream
result_cond = ar_cond.get(timeout=10)
self.assertTrue(result_cond)
def test_create_read_delete_event_process(self):
"""
Test that the CRUD methods for the event processes work correctly
"""
#---------------------------------------------------------------------------------------------
# Create a process definition
#---------------------------------------------------------------------------------------------
# Create
module = 'ion.processes.data.transforms.event_alert_transform'
class_name = 'EventAlertTransform'
procdef_id = self.event_management.create_event_process_definition(version='ver_1',
module=module,
class_name=class_name,
uri='http://hare.com',
arguments=['arg1', 'arg2'],
event_types=['ExampleDetectableEvent', 'type_2'],
sub_types=['sub_type_1'])
# Read
read_process_def = self.event_management.read_event_process_definition(procdef_id)
self.assertEquals(read_process_def.arguments, ['arg1', 'arg2'])
#---------------------------------------------------------------------------------------------
# Use the process definition to create a process
#---------------------------------------------------------------------------------------------
# Create a stream
param_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
stream_def_id = self.pubsub.create_stream_definition('cond_stream_def', parameter_dictionary_id=param_dict_id)
tdom, sdom = time_series_domain()
tdom, sdom = tdom.dump(), sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
# Create a data product
data_product_id = self.data_product_management.create_data_product(data_product=dp_obj, stream_definition_id=stream_def_id)
output_products = {}
output_products['conductivity'] = data_product_id
# Create an event process
event_process_id = self.event_management.create_event_process( process_definition_id=procdef_id,
event_types=['ExampleDetectableEvent','DetectionEvent'],
sub_types=['s1', 's2'],
origins=['or_1', 'or_2'],
origin_types=['or_t1', 'or_t2'],
out_data_products = output_products)
self.addCleanup(self.process_dispatcher.cancel_process, event_process_id)
#---------------------------------------------------------------------------------------------
# Read the event process object and make assertions
#--------------------------------------------------------------------------------------------- out_data_products={'conductivity': data_product_id})
event_process_obj = self.event_management.read_event_process(event_process_id=event_process_id)
# Get the stream associated with the data product for the sake of making assertions
stream_ids, _ = self.rrc.find_objects(data_product_id, PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
# Assertions!
self.assertEquals(event_process_obj.detail.output_streams['conductivity'], stream_id)
self.assertEquals(event_process_obj.detail.event_types, ['ExampleDetectableEvent', 'DetectionEvent'])
self.assertEquals(event_process_obj.detail.sub_types, ['s1', 's2'])
self.assertEquals(event_process_obj.detail.origins, ['or_1', 'or_2'])
self.assertEquals(event_process_obj.detail.origin_types, ['or_t1', 'or_t2'])
class TestDMEnd2End(IonIntegrationTestCase):
def setUp(self): # Love the non pep-8 convention
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.ingestion_management = IngestionManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.event = Event()
self.exchange_space_name = 'test_granules'
self.exchange_point_name = 'science_data'
self.i = 0
self.cci = 0
#--------------------------------------------------------------------------------
# Helper/Utility methods
#--------------------------------------------------------------------------------
def create_dataset(self, parameter_dict_id=''):
'''
Creates a time-series dataset
'''
if not parameter_dict_id:
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
dataset = Dataset('test_dataset_%i'%self.i)
dataset_id = self.dataset_management.create_dataset(dataset, parameter_dictionary_id=parameter_dict_id)
self.addCleanup(self.dataset_management.delete_dataset, dataset_id)
return dataset_id
def get_datastore(self, dataset_id):
'''
Gets an instance of the datastore
This method is primarily used to defeat a bug where integration tests in multiple containers may sometimes
delete a CouchDB datastore and the other containers are unaware of the new state of the datastore.
'''
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def get_ingestion_config(self):
'''
Grab the ingestion configuration from the resource registry
'''
# The ingestion configuration should have been created by the bootstrap service
# which is configured through r2deploy.yml
ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration,id_only=True)
return ingest_configs[0]
def launch_producer(self, stream_id=''):
'''
Launch the producer
'''
pid = self.container.spawn_process('better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process':{'stream_id':stream_id}})
self.addCleanup(self.container.terminate_process, pid)
def make_simple_dataset(self):
'''
Makes a stream, a stream definition and a dataset, the essentials for most of these tests
'''
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd data %i' % self.i, parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('ctd stream %i' % self.i, 'xp1', stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
dataset_id = self.create_dataset(pdict_id)
# self.get_datastore(dataset_id)
self.i += 1
return stream_id, route, stream_def_id, dataset_id
def publish_hifi(self,stream_id,stream_route,offset=0):
'''
Publish deterministic data
'''
pub = StandaloneStreamPublisher(stream_id, stream_route)
stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id)
stream_def_id = stream_def._id
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10) + (offset * 10)
rdt['temp'] = np.arange(10) + (offset * 10)
pub.publish(rdt.to_granule())
def publish_fake_data(self,stream_id, route):
'''
Make four granules
'''
for i in xrange(4):
self.publish_hifi(stream_id,route,i)
def start_ingestion(self, stream_id, dataset_id):
'''
Starts ingestion/persistence for a given dataset
'''
ingest_config_id = self.get_ingestion_config()
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
def stop_ingestion(self, stream_id):
ingest_config_id = self.get_ingestion_config()
self.ingestion_management.unpersist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id)
def validate_granule_subscription(self, msg, route, stream_id):
'''
Validation for granule format
'''
if msg == {}:
return
rdt = RecordDictionaryTool.load_from_granule(msg)
log.info('%s', rdt.pretty_print())
self.assertIsInstance(msg,Granule,'Message is improperly formatted. (%s)' % type(msg))
self.event.set()
def wait_until_we_have_enough_granules(self, dataset_id='',data_size=40):
'''
Loops until there is a sufficient amount of data in the dataset
'''
done = False
with gevent.Timeout(40):
while not done:
extents = self.dataset_management.dataset_extents(dataset_id, 'time')
granule = self.data_retriever.retrieve_last_data_points(dataset_id, 1)
rdt = RecordDictionaryTool.load_from_granule(granule)
if rdt['time'] and rdt['time'][0] != rdt._pdict.get_context('time').fill_value and extents >= data_size:
done = True
else:
gevent.sleep(0.2)
#--------------------------------------------------------------------------------
# Test Methods
#--------------------------------------------------------------------------------
def test_dm_end_2_end(self):
#--------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
#--------------------------------------------------------------------------------
self.event.clear()
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
#--------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
#--------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
#--------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
#--------------------------------------------------------------------------------
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id,40)
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:])
self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all())
#--------------------------------------------------------------------------------
# Now to try the streamed approach
#--------------------------------------------------------------------------------
replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id)
log.info('Process ID: %s', process_id)
replay_client = ReplayClient(process_id)
#--------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
#--------------------------------------------------------------------------------
sub_id = self.pubsub_management.create_subscription(self.exchange_space_name,stream_ids=[replay_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
subscriber.start()
self.addCleanup(subscriber.stop)
self.data_retriever.start_replay_agent(self.replay_id)
self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched')
replay_client.start_replay()
self.assertTrue(self.event.wait(10))
self.data_retriever.cancel_replay_agent(self.replay_id)
#--------------------------------------------------------------------------------
# Test the slicing capabilities
#--------------------------------------------------------------------------------
granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)})
rdt = RecordDictionaryTool.load_from_granule(granule)
b = rdt['time'] == np.arange(5)
self.assertTrue(b.all() if not isinstance(b,bool) else b)
def test_coverage_transform(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id)
self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id)
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = ph.get_rdt(stream_def_id)
ph.fill_parsed_rdt(rdt)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], rdt['time'])
np.testing.assert_array_almost_equal(rdt_out['temp'], rdt['temp'])
np.testing.assert_allclose(rdt_out['conductivity_L1'], np.array([42.914]))
np.testing.assert_allclose(rdt_out['temp_L1'], np.array([20.]))
np.testing.assert_allclose(rdt_out['pressure_L1'], np.array([3.068]))
np.testing.assert_allclose(rdt_out['density'], np.array([1021.7144739593881], dtype='float32'))
np.testing.assert_allclose(rdt_out['salinity'], np.array([30.935132729668283], dtype='float32'))
def test_ingestion_pause(self):
ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
ingestion_config_id = self.get_ingestion_config()
self.start_ingestion(ctd_stream_id, dataset_id)
self.addCleanup(self.stop_ingestion, ctd_stream_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
publisher = StandaloneStreamPublisher(ctd_stream_id, route)
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(monitor.wait())
granule = self.data_retriever.retrieve(dataset_id)
self.ingestion_management.pause_data_stream(ctd_stream_id, ingestion_config_id)
monitor.event.clear()
rdt['time'] = np.arange(10,20)
publisher.publish(rdt.to_granule())
self.assertFalse(monitor.event.wait(1))
self.ingestion_management.resume_data_stream(ctd_stream_id, ingestion_config_id)
self.assertTrue(monitor.wait())
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt2['time'], np.arange(20))
def test_last_granule(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
self.publish_hifi(stream_id,route, 0)
self.publish_hifi(stream_id,route, 1)
self.wait_until_we_have_enough_granules(dataset_id,20) # I just need two
success = False
def verifier():
replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(10) + 10
if not isinstance(comp,bool):
return comp.all()
return False
success = poll(verifier)
self.assertTrue(success)
success = False
def verify_points():
replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id,5)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(15,20)
if not isinstance(comp,bool):
return comp.all()
return False
success = poll(verify_points)
self.assertTrue(success)
def test_replay_with_parameters(self):
#--------------------------------------------------------------------------------
# Create the configurations and the dataset
#--------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.publish_fake_data(stream_id, route)
self.assertTrue(dataset_monitor.wait())
query = {
'start_time': 0 - 2208988800,
'end_time': 19 - 2208988800,
'stride_time' : 2,
'parameters': ['time','temp']
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
np.testing.assert_array_equal(rdt['time'], np.arange(0,20,2))
self.assertEquals(set(rdt.iterkeys()), set(['time','temp']))
extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp'])
self.assertTrue(extents['time']>=20)
self.assertTrue(extents['temp']>=20)
def test_repersist_data(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
self.publish_hifi(stream_id,route,0)
self.publish_hifi(stream_id,route,1)
self.wait_until_we_have_enough_granules(dataset_id,20)
config_id = self.get_ingestion_config()
self.ingestion_management.unpersist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id,dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
self.publish_hifi(stream_id,route,2)
self.publish_hifi(stream_id,route,3)
self.wait_until_we_have_enough_granules(dataset_id,40)
success = False
with gevent.timeout.Timeout(5):
while not success:
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(0,40)
if not isinstance(comp,bool):
success = comp.all()
gevent.sleep(1)
self.assertTrue(success)
@unittest.skip('deprecated')
def test_correct_time(self):
# There are 2208988800 seconds between Jan 1 1900 and Jan 1 1970, i.e.
# the conversion factor between unix and NTP time
unix_now = np.floor(time.time())
ntp_now = unix_now + 2208988800
unix_ago = unix_now - 20
ntp_ago = unix_ago + 2208988800
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
coverage = DatasetManagementService._get_simplex_coverage(dataset_id, mode='a')
coverage.insert_timesteps(20)
coverage.set_parameter_values('time', np.arange(ntp_ago,ntp_now))
temporal_bounds = self.dataset_management.dataset_temporal_bounds(dataset_id)
self.assertTrue( np.abs(temporal_bounds[0] - unix_ago) < 2)
self.assertTrue( np.abs(temporal_bounds[1] - unix_now) < 2)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_out_of_band_retrieve(self):
# Setup the environemnt
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
# Fill the dataset
self.publish_fake_data(stream_id, route)
self.wait_until_we_have_enough_granules(dataset_id,40)
# Retrieve the data
granule = DataRetrieverService.retrieve_oob(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assertTrue((rdt['time'] == np.arange(40)).all())
def publish_and_wait(self, dataset_id, granule):
stream_ids, _ = self.resource_registry.find_objects(dataset_id, PRED.hasStream,id_only=True)
stream_id=stream_ids[0]
route = self.pubsub_management.read_stream_route(stream_id)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(granule)
self.assertTrue(dataset_monitor.wait())
def test_sparse_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_sparse()
stream_def_id = self.pubsub_management.create_stream_definition('sparse', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
dataset_id = self.create_dataset(pdict_id)
self.start_ingestion(stream_id,dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
# Publish initial granule
# the first one has the sparse value set inside it, sets lat to 45 and lon to -71
ntp_now = time.time() + 2208988800
rdt = ph.get_rdt(stream_def_id)
rdt['time'] = [ntp_now]
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [300000]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = ['']
rdt['lat'] = [45]
rdt['conductivity'] = [4341400]
rdt['driver_timestamp'] = [ntp_now]
rdt['lon'] = [-71]
rdt['pressure'] = [256.8]
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
dataset_monitor.reset()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
# Check the values and make sure they're correct
np.testing.assert_allclose(rdt_out['time'], rdt['time'])
np.testing.assert_allclose(rdt_out['temp'], rdt['temp'])
np.testing.assert_allclose(rdt_out['lat'], np.array([45]))
np.testing.assert_allclose(rdt_out['lon'], np.array([-71]))
np.testing.assert_allclose(rdt_out['conductivity_L1'], np.array([42.914]))
np.testing.assert_allclose(rdt_out['temp_L1'], np.array([20.]))
np.testing.assert_allclose(rdt_out['pressure_L1'], np.array([3.068]))
np.testing.assert_allclose(rdt_out['density'], np.array([1021.7144739593881], dtype='float32'))
np.testing.assert_allclose(rdt_out['salinity'], np.array([30.935132729668283], dtype='float32'))
# We're going to change the lat/lon
rdt = ph.get_rdt(stream_def_id)
rdt['time'] = time.time() + 2208988800
rdt['lat'] = [46]
rdt['lon'] = [-73]
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
dataset_monitor.reset()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_allclose(rdt_out['time'], rdt['time'])
for i in xrange(9):
ntp_now = time.time() + 2208988800
rdt['time'] = [ntp_now]
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [300000]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = [None]
rdt['conductivity'] = [4341400]
rdt['driver_timestamp'] = [ntp_now]
rdt['pressure'] = [256.8]
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
dataset_monitor.reset()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_allclose(rdt_out['pressure'], np.array([256.8] * 10))
np.testing.assert_allclose(rdt_out['lat'], np.array([45] + [46] * 9))
np.testing.assert_allclose(rdt_out['lon'], np.array([-71] + [-73] * 9))
class TestTransformPrime(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Because hey why not?!
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
def setup_streams(self):
in_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('sbe37_L0_test', id_only=True)
out_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('sbe37_L1_test', id_only=True)
in_stream_def_id = self.pubsub_management.create_stream_definition('L0 SBE37', parameter_dictionary_id=in_pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, in_stream_def_id)
out_stream_def_id = self.pubsub_management.create_stream_definition('L1 SBE37', parameter_dictionary_id=out_pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, out_stream_def_id)
in_stream_id, in_route = self.pubsub_management.create_stream('L0 input', stream_definition_id=in_stream_def_id, exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, in_stream_id)
out_stream_id, out_route = self.pubsub_management.create_stream('L0 output', stream_definition_id=out_stream_def_id, exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, out_stream_id)
return [(in_stream_id, in_stream_def_id), (out_stream_id, out_stream_def_id)]
def setup_advanced_streams(self):
in_pdict_id = out_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('sbe37_LC_TEST', id_only=True)
in_stream_def_id = self.pubsub_management.create_stream_definition('sbe37_instrument', parameter_dictionary_id=in_pdict_id, available_fields=['time', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0', 'lat', 'lon'])
self.addCleanup(self.pubsub_management.delete_stream_definition, in_stream_def_id)
out_stream_def_id = self.pubsub_management.create_stream_definition('sbe37_l2', parameter_dictionary_id=out_pdict_id, available_fields=['time', 'rho','PRACSAL_L2'])
self.addCleanup(self.pubsub_management.delete_stream_definition, out_stream_def_id)
in_stream_id, in_route = self.pubsub_management.create_stream('instrument stream', stream_definition_id=in_stream_def_id, exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, in_stream_id)
out_stream_id, out_route = self.pubsub_management.create_stream('data product stream', stream_definition_id=out_stream_def_id, exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, out_stream_id)
return [(in_stream_id, in_stream_def_id), (out_stream_id, out_stream_def_id)]
def preload(self):
config = DotDict()
config.op = 'load'
config.scenario = 'BASE,LC_TEST'
config.categories = 'ParameterFunctions,ParameterDefs,ParameterDictionary'
config.path = 'res/preload/r2_ioc'
self.container.spawn_process('preload','ion.processes.bootstrap.ion_loader','IONLoader', config)
def setup_advanced_transform(self):
self.preload()
queue_name = 'transform_prime'
stream_info = self.setup_advanced_streams()
in_stream_id, in_stream_def_id = stream_info[0]
out_stream_id, out_stream_def_id = stream_info[1]
routes = {}
routes[(in_stream_id, out_stream_id)]= None
config = DotDict()
config.process.queue_name = queue_name
config.process.routes = routes
config.process.publish_streams = {out_stream_id:out_stream_id}
sub_id = self.pubsub_management.create_subscription(queue_name, stream_ids=[in_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
self.container.spawn_process('transform_prime', 'ion.processes.data.transforms.transform_prime','TransformPrime', config)
listen_sub_id = self.pubsub_management.create_subscription('listener', stream_ids=[out_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, listen_sub_id)
self.pubsub_management.activate_subscription(listen_sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, listen_sub_id)
return [(in_stream_id, in_stream_def_id), (out_stream_id, out_stream_def_id)]
def setup_transform(self):
self.preload()
queue_name = 'transform_prime'
stream_info = self.setup_streams()
in_stream_id, in_stream_def_id = stream_info[0]
out_stream_id, out_stream_def_id = stream_info[1]
routes = {}
routes[(in_stream_id, out_stream_id)]= None
config = DotDict()
config.process.queue_name = queue_name
config.process.routes = routes
config.process.publish_streams = {out_stream_id:out_stream_id}
sub_id = self.pubsub_management.create_subscription(queue_name, stream_ids=[in_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
self.container.spawn_process('transform_prime', 'ion.processes.data.transforms.transform_prime','TransformPrime', config)
listen_sub_id = self.pubsub_management.create_subscription('listener', stream_ids=[out_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, listen_sub_id)
self.pubsub_management.activate_subscription(listen_sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, listen_sub_id)
return [(in_stream_id, in_stream_def_id), (out_stream_id, out_stream_def_id)]
def setup_validator(self, validator):
listener = StandaloneStreamSubscriber('listener', validator)
listener.start()
self.addCleanup(listener.stop)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_execute_advanced_transform(self):
# Runs a transform across L0-L2 with stream definitions including available fields
streams = self.setup_advanced_transform()
in_stream_id, in_stream_def_id = streams[0]
out_stream_id, out_stream_defs_id = streams[1]
validation_event = Event()
def validator(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
if not np.allclose(rdt['rho'], np.array([1001.0055034])):
return
validation_event.set()
self.setup_validator(validator)
in_route = self.pubsub_management.read_stream_route(in_stream_id)
publisher = StandaloneStreamPublisher(in_stream_id, in_route)
outbound_rdt = RecordDictionaryTool(stream_definition_id=in_stream_def_id)
outbound_rdt['time'] = [0]
outbound_rdt['TEMPWAT_L0'] = [280000]
outbound_rdt['CONDWAT_L0'] = [100000]
outbound_rdt['PRESWAT_L0'] = [2789]
outbound_rdt['lat'] = [45]
outbound_rdt['lon'] = [-71]
outbound_granule = outbound_rdt.to_granule()
publisher.publish(outbound_granule)
self.assertTrue(validation_event.wait(2))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_execute_transform(self):
streams = self.setup_transform()
in_stream_id, in_stream_def_id = streams[0]
out_stream_id, out_stream_def_id = streams[1]
validation_event = Event()
def validator(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
if not np.allclose(rdt['TEMPWAT_L1'], np.array([18.])):
return
if not np.allclose(rdt['CONDWAT_L1'], np.array([0.5])):
return
if not np.allclose(rdt['PRESWAT_L1'], np.array([0.04536611])):
return
validation_event.set()
self.setup_validator(validator)
in_route = self.pubsub_management.read_stream_route(in_stream_id)
publisher = StandaloneStreamPublisher(in_stream_id, in_route)
outbound_rdt = RecordDictionaryTool(stream_definition_id=in_stream_def_id)
outbound_rdt['time'] = [0]
outbound_rdt['TEMPWAT_L0'] = [280000]
outbound_rdt['CONDWAT_L0'] = [100000]
outbound_rdt['PRESWAT_L0'] = [2789]
outbound_rdt['lat'] = [45]
outbound_rdt['lon'] = [-71]
outbound_granule = outbound_rdt.to_granule()
publisher.publish(outbound_granule)
self.assertTrue(validation_event.wait(2))
def instrument_test_driver(container):
org_client = OrgManagementServiceClient(node=container.node)
id_client = IdentityManagementServiceClient(node=container.node)
system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor)
log.info('system actor:' + system_actor._id)
sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id))
# Names of agent data streams to be configured.
parsed_stream_name = 'ctd_parsed'
raw_stream_name = 'ctd_raw'
# Driver configuration.
#Simulator
driver_config = {
'svr_addr': 'localhost',
'cmd_port': 5556,
'evt_port': 5557,
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'comms_config': {
SBE37Channel.CTD: {
'method':'ethernet',
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'server_addr': 'localhost',
'server_port': 8888
}
}
}
#Hardware
_container_client = ContainerAgentClient(node=container.node,
name=container.name)
# Create a pubsub client to create streams.
_pubsub_client = PubsubManagementServiceClient(node=container.node)
# A callback for processing subscribed-to data.
def consume(message, headers):
log.info('Subscriber received message: %s', str(message))
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=container,
node=container.node)
subs = []
# Create streams for each stream named in driver.
stream_config = {}
for (stream_name, val) in PACKET_CONFIG.iteritems():
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = _pubsub_client.create_stream_definition(
container=stream_def)
stream_id = _pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': sa_header_roles })
stream_config[stream_name] = stream_id
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume)
sub.start()
query = StreamQuery(stream_ids=[stream_id])
sub_id = _pubsub_client.create_subscription(\
query=query, exchange_name=exchange_name )
_pubsub_client.activate_subscription(sub_id)
subs.append(sub)
# Create agent config.
agent_resource_id = '123xyz'
agent_config = {
'driver_config' : driver_config,
'stream_config' : stream_config,
'agent' : {'resource_id': agent_resource_id}
}
# Launch an instrument agent process.
_ia_name = 'agent007'
_ia_mod = 'ion.agents.instrument.instrument_agent'
_ia_class = 'InstrumentAgent'
_ia_pid = _container_client.spawn_process(name=_ia_name,
module=_ia_mod, cls=_ia_class,
config=agent_config)
log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
class TestDMEnd2End(IonIntegrationTestCase):
def setUp(self): # Love the non pep-8 convention
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.ingestion_management = IngestionManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.pids = []
self.event = Event()
self.exchange_space_name = 'test_granules'
self.exchange_point_name = 'science_data'
self.i = 0
self.purge_queues()
self.queue_buffer = []
self.streams = []
self.addCleanup(self.stop_all_ingestion)
def purge_queues(self):
xn = self.container.ex_manager.create_xn_queue('science_granule_ingestion')
xn.purge()
def tearDown(self):
self.purge_queues()
for pid in self.pids:
self.container.proc_manager.terminate_process(pid)
IngestionManagementIntTest.clean_subscriptions()
for queue in self.queue_buffer:
if isinstance(queue, ExchangeNameQueue):
queue.delete()
elif isinstance(queue, str):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
#--------------------------------------------------------------------------------
# Helper/Utility methods
#--------------------------------------------------------------------------------
def create_dataset(self, parameter_dict_id=''):
'''
Creates a time-series dataset
'''
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
if not parameter_dict_id:
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
dataset_id = self.dataset_management.create_dataset('test_dataset_%i'%self.i, parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom)
return dataset_id
def get_datastore(self, dataset_id):
'''
Gets an instance of the datastore
This method is primarily used to defeat a bug where integration tests in multiple containers may sometimes
delete a CouchDB datastore and the other containers are unaware of the new state of the datastore.
'''
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def get_ingestion_config(self):
'''
Grab the ingestion configuration from the resource registry
'''
# The ingestion configuration should have been created by the bootstrap service
# which is configured through r2deploy.yml
ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration,id_only=True)
return ingest_configs[0]
def launch_producer(self, stream_id=''):
'''
Launch the producer
'''
pid = self.container.spawn_process('better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process':{'stream_id':stream_id}})
self.pids.append(pid)
def make_simple_dataset(self):
'''
Makes a stream, a stream definition and a dataset, the essentials for most of these tests
'''
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('ctd stream %i' % self.i, 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
self.get_datastore(dataset_id)
self.i += 1
return stream_id, route, stream_def_id, dataset_id
def publish_hifi(self,stream_id,stream_route,offset=0):
'''
Publish deterministic data
'''
pub = StandaloneStreamPublisher(stream_id, stream_route)
stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id)
stream_def_id = stream_def._id
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10) + (offset * 10)
rdt['temp'] = np.arange(10) + (offset * 10)
pub.publish(rdt.to_granule())
def publish_fake_data(self,stream_id, route):
'''
Make four granules
'''
for i in xrange(4):
self.publish_hifi(stream_id,route,i)
def start_ingestion(self, stream_id, dataset_id):
'''
Starts ingestion/persistence for a given dataset
'''
ingest_config_id = self.get_ingestion_config()
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
def stop_ingestion(self, stream_id):
ingest_config_id = self.get_ingestion_config()
self.ingestion_management.unpersist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id)
def stop_all_ingestion(self):
try:
[self.stop_ingestion(sid) for sid in self.streams]
except:
pass
def validate_granule_subscription(self, msg, route, stream_id):
'''
Validation for granule format
'''
if msg == {}:
return
rdt = RecordDictionaryTool.load_from_granule(msg)
log.info('%s', rdt.pretty_print())
self.assertIsInstance(msg,Granule,'Message is improperly formatted. (%s)' % type(msg))
self.event.set()
def wait_until_we_have_enough_granules(self, dataset_id='',data_size=40):
'''
Loops until there is a sufficient amount of data in the dataset
'''
done = False
with gevent.Timeout(40):
while not done:
extents = self.dataset_management.dataset_extents(dataset_id, 'time')[0]
granule = self.data_retriever.retrieve_last_data_points(dataset_id, 1)
rdt = RecordDictionaryTool.load_from_granule(granule)
if rdt['time'] and rdt['time'][0] != rdt._pdict.get_context('time').fill_value and extents >= data_size:
done = True
else:
gevent.sleep(0.2)
#--------------------------------------------------------------------------------
# Test Methods
#--------------------------------------------------------------------------------
@attr('SMOKE')
def test_dm_end_2_end(self):
#--------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
#--------------------------------------------------------------------------------
self.event.clear()
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
#--------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
#--------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
#--------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
#--------------------------------------------------------------------------------
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id,40)
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:])
self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all())
#--------------------------------------------------------------------------------
# Now to try the streamed approach
#--------------------------------------------------------------------------------
replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id)
log.info('Process ID: %s', process_id)
replay_client = ReplayClient(process_id)
#--------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
#--------------------------------------------------------------------------------
xp = self.container.ex_manager.create_xp(self.exchange_point_name)
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
self.data_retriever.start_replay_agent(self.replay_id)
self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched')
replay_client.start_replay()
self.assertTrue(self.event.wait(10))
subscriber.stop()
self.data_retriever.cancel_replay_agent(self.replay_id)
#--------------------------------------------------------------------------------
# Test the slicing capabilities
#--------------------------------------------------------------------------------
granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)})
rdt = RecordDictionaryTool.load_from_granule(granule)
b = rdt['time'] == np.arange(5)
self.assertTrue(b.all() if not isinstance(b,bool) else b)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
def test_coverage_transform(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id)
self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id)
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = ph.get_rdt(stream_def_id)
ph.fill_parsed_rdt(rdt)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], rdt['time'])
np.testing.assert_array_almost_equal(rdt_out['temp'], rdt['temp'])
np.testing.assert_array_almost_equal(rdt_out['conductivity_L1'], np.array([42.914]))
np.testing.assert_array_almost_equal(rdt_out['temp_L1'], np.array([20.]))
np.testing.assert_array_almost_equal(rdt_out['pressure_L1'], np.array([3.068]))
np.testing.assert_array_almost_equal(rdt_out['density'], np.array([1021.7144739593881]))
np.testing.assert_array_almost_equal(rdt_out['salinity'], np.array([30.935132729668283]))
def test_qc_events(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_qc_pdict()
stream_def_id = self.pubsub_management.create_stream_definition('qc stream def', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('qc stream', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
config = DotDict()
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id, config=config)
self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id)
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.arange(10) * 3
verified = Event()
def verification(event, *args, **kwargs):
self.assertEquals(event.qc_parameter, 'temp_qc')
self.assertEquals(event.temporal_value, 7)
verified.set()
es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=dataset_id, callback=verification, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(verified.wait(10))
def test_lookup_values_ingest_replay(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsub_management.create_stream_definition('lookups', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
config = DotDict()
config.process.lookup_docs = ['test1', 'test2']
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id, config=config)
self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id)
stored_value_manager = StoredValueManager(self.container)
stored_value_manager.stored_value_cas('test1',{'offset_a':10.0, 'offset_b':13.1})
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = [20.0] * 20
granule = rdt.to_granule()
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(20))
np.testing.assert_array_almost_equal(rdt_out['temp'], np.array([20.] * 20))
np.testing.assert_array_almost_equal(rdt_out['calibrated'], np.array([30.]*20))
np.testing.assert_array_equal(rdt_out['offset_b'], np.array([rdt_out.fill_value('offset_b')] * 20))
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(20,40)
rdt['temp'] = [20.0] * 20
granule = rdt.to_granule()
dataset_monitor.event.clear()
stored_value_manager.stored_value_cas('test1',{'offset_a':20.0})
stored_value_manager.stored_value_cas('coefficient_document',{'offset_b':10.0})
gevent.sleep(2)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(40))
np.testing.assert_array_almost_equal(rdt_out['temp'], np.array([20.] * 20 + [20.] * 20))
np.testing.assert_array_equal(rdt_out['offset_b'], np.array([10.] * 40))
np.testing.assert_array_almost_equal(rdt_out['calibrated'], np.array([30.]*20 + [40.]*20))
np.testing.assert_array_almost_equal(rdt_out['calibrated_b'], np.array([40.] * 20 + [50.] * 20))
@unittest.skip('Doesnt work')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_replay_pause(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
scov = DatasetManagementService._get_simplex_coverage(dataset_id)
bb = CoverageCraft(scov)
bb.rdt['time'] = np.arange(100)
bb.rdt['temp'] = np.random.random(100) + 30
bb.sync_with_granule()
DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations
# Set up the subscriber to verify the data
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
xp = self.container.ex_manager.create_xp('xp1')
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
# Set up the replay agent and the client wrapper
# 1) Define the Replay (dataset and stream to publish on)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream)
# 2) Make a client to the interact with the process (optionall provide it a process to bind with)
replay_client = ReplayClient(process_id)
# 3) Start the agent (launch the process)
self.data_retriever.start_replay_agent(self.replay_id)
# 4) Start replaying...
replay_client.start_replay()
# Wait till we get some granules
self.assertTrue(self.event.wait(5))
# We got granules, pause the replay, clear the queue and allow the process to finish consuming
replay_client.pause_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure there's no remaining messages being consumed
self.assertFalse(self.event.wait(1))
# Resume the replay and wait until we start getting granules again
replay_client.resume_replay()
self.assertTrue(self.event.wait(5))
# Stop the replay, clear the queues
replay_client.stop_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure that it did indeed stop
self.assertFalse(self.event.wait(1))
subscriber.stop()
def test_retrieve_and_transform(self):
# Make a simple dataset and start ingestion, pretty standard stuff.
ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(ctd_stream_id, dataset_id)
# Stream definition for the salinity data
salinity_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
sal_stream_def_id = self.pubsub_management.create_stream_definition('sal data', parameter_dictionary_id=salinity_pdict_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.random.randn(10) * 10 + 30
rdt['conductivity'] = np.random.randn(10) * 2 + 10
rdt['pressure'] = np.random.randn(10) * 1 + 12
publisher = StandaloneStreamPublisher(ctd_stream_id, route)
publisher.publish(rdt.to_granule())
rdt['time'] = np.arange(10,20)
publisher.publish(rdt.to_granule())
self.wait_until_we_have_enough_granules(dataset_id, 20)
granule = self.data_retriever.retrieve(dataset_id,
None,
None,
'ion.processes.data.transforms.ctd.ctd_L2_salinity',
'CTDL2SalinityTransformAlgorithm',
kwargs=dict(params=sal_stream_def_id))
rdt = RecordDictionaryTool.load_from_granule(granule)
for i in rdt['salinity']:
self.assertNotEquals(i,0)
self.streams.append(ctd_stream_id)
self.stop_ingestion(ctd_stream_id)
def test_last_granule(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
self.publish_hifi(stream_id,route, 0)
self.publish_hifi(stream_id,route, 1)
self.wait_until_we_have_enough_granules(dataset_id,20) # I just need two
success = False
def verifier():
replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(10) + 10
if not isinstance(comp,bool):
return comp.all()
return False
success = poll(verifier)
self.assertTrue(success)
success = False
def verify_points():
replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id,5)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(15,20)
if not isinstance(comp,bool):
return comp.all()
return False
success = poll(verify_points)
self.assertTrue(success)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
def test_replay_with_parameters(self):
#--------------------------------------------------------------------------------
# Create the configurations and the dataset
#--------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.publish_fake_data(stream_id, route)
self.assertTrue(dataset_monitor.event.wait(30))
query = {
'start_time': 0 - 2208988800,
'end_time': 20 - 2208988800,
'stride_time' : 2,
'parameters': ['time','temp']
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
comp = np.arange(0,20,2) == rdt['time']
self.assertTrue(comp.all(),'%s' % rdt.pretty_print())
self.assertEquals(set(rdt.iterkeys()), set(['time','temp']))
extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp'])
self.assertTrue(extents['time']>=20)
self.assertTrue(extents['temp']>=20)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
def test_repersist_data(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
self.publish_hifi(stream_id,route,0)
self.publish_hifi(stream_id,route,1)
self.wait_until_we_have_enough_granules(dataset_id,20)
config_id = self.get_ingestion_config()
self.ingestion_management.unpersist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id,dataset_id=dataset_id)
self.publish_hifi(stream_id,route,2)
self.publish_hifi(stream_id,route,3)
self.wait_until_we_have_enough_granules(dataset_id,40)
success = False
with gevent.timeout.Timeout(5):
while not success:
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(0,40)
if not isinstance(comp,bool):
success = comp.all()
gevent.sleep(1)
self.assertTrue(success)
self.streams.append(stream_id)
self.stop_ingestion(stream_id)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_correct_time(self):
# There are 2208988800 seconds between Jan 1 1900 and Jan 1 1970, i.e.
# the conversion factor between unix and NTP time
unix_now = np.floor(time.time())
ntp_now = unix_now + 2208988800
unix_ago = unix_now - 20
ntp_ago = unix_ago + 2208988800
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
coverage = DatasetManagementService._get_simplex_coverage(dataset_id)
coverage.insert_timesteps(20)
coverage.set_parameter_values('time', np.arange(ntp_ago,ntp_now))
temporal_bounds = self.dataset_management.dataset_temporal_bounds(dataset_id)
self.assertTrue( np.abs(temporal_bounds[0] - unix_ago) < 2)
self.assertTrue( np.abs(temporal_bounds[1] - unix_now) < 2)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_empty_coverage_time(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
coverage = DatasetManagementService._get_coverage(dataset_id)
temporal_bounds = self.dataset_management.dataset_temporal_bounds(dataset_id)
self.assertEquals([coverage.get_parameter_context('time').fill_value] *2, temporal_bounds)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_out_of_band_retrieve(self):
# Setup the environemnt
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
# Fill the dataset
self.publish_fake_data(stream_id, route)
self.wait_until_we_have_enough_granules(dataset_id,40)
# Retrieve the data
granule = DataRetrieverService.retrieve_oob(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assertTrue((rdt['time'] == np.arange(40)).all())
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_retrieve_cache(self):
DataRetrieverService._refresh_interval = 1
datasets = [self.make_simple_dataset() for i in xrange(10)]
for stream_id, route, stream_def_id, dataset_id in datasets:
coverage = DatasetManagementService._get_simplex_coverage(dataset_id)
coverage.insert_timesteps(10)
coverage.set_parameter_values('time', np.arange(10))
coverage.set_parameter_values('temp', np.arange(10))
# Verify cache hit and refresh
dataset_ids = [i[3] for i in datasets]
self.assertTrue(dataset_ids[0] not in DataRetrieverService._retrieve_cache)
DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache
cov, age = DataRetrieverService._retrieve_cache[dataset_ids[0]]
# Verify that it was hit and it's now in there
self.assertTrue(dataset_ids[0] in DataRetrieverService._retrieve_cache)
gevent.sleep(DataRetrieverService._refresh_interval + 0.2)
DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache
cov, age2 = DataRetrieverService._retrieve_cache[dataset_ids[0]]
self.assertTrue(age2 != age)
for dataset_id in dataset_ids:
DataRetrieverService._get_coverage(dataset_id)
self.assertTrue(dataset_ids[0] not in DataRetrieverService._retrieve_cache)
stream_id, route, stream_def, dataset_id = datasets[0]
self.start_ingestion(stream_id, dataset_id)
DataRetrieverService._get_coverage(dataset_id)
self.assertTrue(dataset_id in DataRetrieverService._retrieve_cache)
DataRetrieverService._refresh_interval = 100
self.publish_hifi(stream_id,route,1)
self.wait_until_we_have_enough_granules(dataset_id, data_size=20)
event = gevent.event.Event()
with gevent.Timeout(20):
while not event.wait(0.1):
if dataset_id not in DataRetrieverService._retrieve_cache:
event.set()
self.assertTrue(event.is_set())
def publish_and_wait(self, dataset_id, granule):
stream_ids, _ = self.resource_registry.find_objects(dataset_id, PRED.hasStream,id_only=True)
stream_id=stream_ids[0]
route = self.pubsub_management.read_stream_route(stream_id)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_monitor = DatasetMonitor(dataset_id)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_thorough_gap_analysis(self):
dataset_id = self.test_ingestion_gap_analysis()
vcov = DatasetManagementService._get_coverage(dataset_id)
self.assertIsInstance(vcov,ViewCoverage)
ccov = vcov.reference_coverage
self.assertIsInstance(ccov, ComplexCoverage)
self.assertEquals(len(ccov._reference_covs), 3)
def test_ingestion_gap_analysis(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
connection1 = uuid4().hex
connection2 = uuid4().hex
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [0]
self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection1,connection_index='0'))
rdt['time'] = [1]
rdt['temp'] = [1]
self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection1,connection_index=1))
rdt['time'] = [2]
rdt['temp'] = [2]
self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection1,connection_index='3')) # Gap, missed message
rdt['time'] = [3]
rdt['temp'] = [3]
self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection2,connection_index='3')) # Gap, new connection
rdt['time'] = [4]
rdt['temp'] = [4]
self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection2,connection_index='4'))
rdt['time'] = [5]
rdt['temp'] = [5]
self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection2,connection_index=5))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(6))
np.testing.assert_array_equal(rdt['temp'], np.arange(6))
return dataset_id
@unittest.skip('Outdated due to ingestion retry')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_ingestion_failover(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
event = Event()
def cb(*args, **kwargs):
event.set()
sub = EventSubscriber(event_type="ExceptionEvent", callback=cb, origin="stream_exception")
sub.start()
self.publish_fake_data(stream_id, route)
self.wait_until_we_have_enough_granules(dataset_id, 40)
file_path = DatasetManagementService._get_coverage_path(dataset_id)
master_file = os.path.join(file_path, '%s_master.hdf5' % dataset_id)
with open(master_file, 'w') as f:
f.write('this will crash HDF')
self.publish_hifi(stream_id, route, 5)
self.assertTrue(event.wait(10))
sub.stop()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_coverage_types(self):
# Make a simple dataset and start ingestion, pretty standard stuff.
ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
cov = DatasetManagementService._get_coverage(dataset_id=dataset_id)
self.assertIsInstance(cov, ViewCoverage)
cov = DatasetManagementService._get_simplex_coverage(dataset_id=dataset_id)
self.assertIsInstance(cov, SimplexCoverage)
class CtdTransformsIntTest(IonIntegrationTestCase):
def setUp(self):
super(CtdTransformsIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.queue_cleanup = []
self.exchange_cleanup = []
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.exchange_name = 'ctd_L0_all_queue'
self.exchange_point = 'test_exchange'
self.i = 0
def tearDown(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
for exchange in self.exchange_cleanup:
xp = self.container.ex_manager.create_xp(exchange)
xp.delete()
def test_ctd_L0_all(self):
'''
Test that packets are processed by the ctd_L0_all transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='ctd_L0_all',
description='For testing ctd_L0_all')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.ctd_L0_all'
process_definition.executable['class'] = 'ctd_L0_all'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('ctd_all_stream_def', parameter_dictionary_id=pdict_id)
cond_stream_id, _ = self.pubsub.create_stream('test_cond',
exchange_point='science_data',
stream_definition_id=stream_def_id)
pres_stream_id, _ = self.pubsub.create_stream('test_pres',
exchange_point='science_data',
stream_definition_id=stream_def_id)
temp_stream_id, _ = self.pubsub.create_stream('test_temp',
exchange_point='science_data',
stream_definition_id=stream_def_id)
config.process.publish_streams.conductivity = cond_stream_id
config.process.publish_streams.pressure = pres_stream_id
config.process.publish_streams.temperature = temp_stream_id
# Schedule the process
pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscribers that will receive the conductivity, temperature and pressure granules from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber1(m, r, s):
ar_cond.set(m)
sub_cond = StandaloneStreamSubscriber('sub_cond', subscriber1)
self.addCleanup(sub_cond.stop)
ar_temp = gevent.event.AsyncResult()
def subscriber2(m,r,s):
ar_temp.set(m)
sub_temp = StandaloneStreamSubscriber('sub_temp', subscriber2)
self.addCleanup(sub_temp.stop)
ar_pres = gevent.event.AsyncResult()
def subscriber3(m,r,s):
ar_pres.set(m)
sub_pres = StandaloneStreamSubscriber('sub_pres', subscriber3)
self.addCleanup(sub_pres.stop)
sub_cond_id= self.pubsub.create_subscription('subscription_cond',
stream_ids=[cond_stream_id],
exchange_name='sub_cond')
sub_temp_id = self.pubsub.create_subscription('subscription_temp',
stream_ids=[temp_stream_id],
exchange_name='sub_temp')
sub_pres_id = self.pubsub.create_subscription('subscription_pres',
stream_ids=[pres_stream_id],
exchange_name='sub_pres')
self.pubsub.activate_subscription(sub_cond_id)
self.pubsub.activate_subscription(sub_temp_id)
self.pubsub.activate_subscription(sub_pres_id)
self.queue_cleanup.append(sub_cond.xn.queue)
self.queue_cleanup.append(sub_temp.xn.queue)
self.queue_cleanup.append(sub_pres.xn.queue)
sub_cond.start()
sub_temp.start()
sub_pres.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result_cond = ar_cond.get(timeout=10)
result_temp = ar_temp.get(timeout=10)
result_pres = ar_pres.get(timeout=10)
out_dict = {}
out_dict['c'] = RecordDictionaryTool.load_from_granule(result_cond)['conductivity']
out_dict['t'] = RecordDictionaryTool.load_from_granule(result_temp)['temp']
out_dict['p'] = RecordDictionaryTool.load_from_granule(result_pres)['pressure']
# Check that the transform algorithm was successfully executed
self.check_granule_splitting(publish_granule, out_dict)
def test_ctd_L1_conductivity(self):
'''
Test that packets are processed by the ctd_L1_conductivity transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='CTDL1ConductivityTransform',
description='For testing CTDL1ConductivityTransform')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.ctd_L1_conductivity'
process_definition.executable['class'] = 'CTDL1ConductivityTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
stream_def_id = self.pubsub.create_stream_definition('cond_stream_def', parameter_dictionary_id=pdict_id)
cond_stream_id, _ = self.pubsub.create_stream('test_conductivity',
exchange_point='science_data',
stream_definition_id=stream_def_id)
config.process.publish_streams.conductivity = cond_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscribers that will receive the conductivity, temperature and pressure granules from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber1(m, r, s):
ar_cond.set(m)
sub_cond = StandaloneStreamSubscriber('sub_cond', subscriber1)
self.addCleanup(sub_cond.stop)
sub_cond_id = self.pubsub.create_subscription('subscription_cond',
stream_ids=[cond_stream_id],
exchange_name='sub_cond')
self.pubsub.activate_subscription(sub_cond_id)
self.queue_cleanup.append(sub_cond.xn.queue)
sub_cond.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result_cond = ar_cond.get(timeout=10)
self.assertTrue(isinstance(result_cond, Granule))
rdt = RecordDictionaryTool.load_from_granule(result_cond)
self.assertTrue(rdt.__contains__('conductivity'))
self.check_cond_algorithm_execution(publish_granule, result_cond)
def check_cond_algorithm_execution(self, publish_granule, granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
output_data = output_rdt_transform['conductivity']
input_data = input_rdt_to_transform['conductivity']
self.assertTrue(((input_data / 100000.0) - 0.5).all() == output_data.all())
def check_pres_algorithm_execution(self, publish_granule, granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
output_data = output_rdt_transform['pressure']
input_data = input_rdt_to_transform['pressure']
self.assertTrue(input_data.all() == output_data.all())
def check_temp_algorithm_execution(self, publish_granule, granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
output_data = output_rdt_transform['temp']
input_data = input_rdt_to_transform['temp']
self.assertTrue(((input_data / 10000.0) - 10).all() == output_data.all())
def check_density_algorithm_execution(self, publish_granule, granule_from_transform):
#------------------------------------------------------------------
# Calculate the correct density from the input granule data
#------------------------------------------------------------------
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
conductivity = input_rdt_to_transform['conductivity']
pressure = input_rdt_to_transform['pressure']
temperature = input_rdt_to_transform['temp']
longitude = input_rdt_to_transform['lon']
latitude = input_rdt_to_transform['lat']
sp = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure)
sa = SA_from_SP(sp, pressure, longitude, latitude)
dens_value = rho(sa, temperature, pressure)
out_density = output_rdt_transform['density']
#-----------------------------------------------------------------------------
# Check that the output data from the transform has the correct density values
#-----------------------------------------------------------------------------
self.assertTrue(dens_value.all() == out_density.all())
def check_salinity_algorithm_execution(self, publish_granule, granule_from_transform):
#------------------------------------------------------------------
# Calculate the correct density from the input granule data
#------------------------------------------------------------------
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
conductivity = input_rdt_to_transform['conductivity']
pressure = input_rdt_to_transform['pressure']
temperature = input_rdt_to_transform['temp']
sal_value = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure)
out_salinity = output_rdt_transform['salinity']
#-----------------------------------------------------------------------------
# Check that the output data from the transform has the correct density values
#-----------------------------------------------------------------------------
self.assertTrue(sal_value.all() == out_salinity.all())
def check_granule_splitting(self, publish_granule, out_dict):
'''
This checks that the ctd_L0_all transform is able to split out one of the
granules from the whole granule
fed into the transform
'''
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
in_cond = input_rdt_to_transform['conductivity']
in_pressure = input_rdt_to_transform['pressure']
in_temp = input_rdt_to_transform['temp']
out_cond = out_dict['c']
out_pres = out_dict['p']
out_temp = out_dict['t']
self.assertTrue(in_cond.all() == out_cond.all())
self.assertTrue(in_pressure.all() == out_pres.all())
self.assertTrue(in_temp.all() == out_temp.all())
def test_ctd_L1_pressure(self):
'''
Test that packets are processed by the ctd_L1_pressure transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='CTDL1PressureTransform',
description='For testing CTDL1PressureTransform')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.ctd_L1_pressure'
process_definition.executable['class'] = 'CTDL1PressureTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('pres_stream_def', parameter_dictionary_id=pdict_id)
pres_stream_id, _ = self.pubsub.create_stream('test_pressure',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.pressure = pres_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscribers that will receive the pressure granules from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_pres = gevent.event.AsyncResult()
def subscriber3(m,r,s):
ar_pres.set(m)
sub_pres = StandaloneStreamSubscriber('sub_pres', subscriber3)
self.addCleanup(sub_pres.stop)
sub_pres_id = self.pubsub.create_subscription('subscription_pres',
stream_ids=[pres_stream_id],
exchange_name='sub_pres')
self.pubsub.activate_subscription(sub_pres_id)
self.queue_cleanup.append(sub_pres.xn.queue)
sub_pres.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_pres.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('pressure'))
self.check_pres_algorithm_execution(publish_granule, result)
def test_ctd_L1_temperature(self):
'''
Test that packets are processed by the ctd_L1_temperature transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='CTDL1TemperatureTransform',
description='For testing CTDL1TemperatureTransform')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.ctd_L1_temperature'
process_definition.executable['class'] = 'CTDL1TemperatureTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('temp_stream_def', parameter_dictionary_id=pdict_id)
temp_stream_id, _ = self.pubsub.create_stream('test_temperature', stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.temperature = temp_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
#---------------------------------------------------------------------------------------------
# Create subscriber that will receive the temperature granule from
# the ctd transform
#---------------------------------------------------------------------------------------------
ar_temp = gevent.event.AsyncResult()
def subscriber2(m,r,s):
ar_temp.set(m)
sub_temp = StandaloneStreamSubscriber('sub_temp', subscriber2)
self.addCleanup(sub_temp.stop)
sub_temp_id = self.pubsub.create_subscription('subscription_temp',
stream_ids=[temp_stream_id],
exchange_name='sub_temp')
self.pubsub.activate_subscription(sub_temp_id)
self.queue_cleanup.append(sub_temp.xn.queue)
sub_temp.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_temp.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('temp'))
self.check_temp_algorithm_execution(publish_granule, result)
def test_ctd_L2_density(self):
'''
Test that packets are processed by the ctd_L1_density transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='DensityTransform',
description='For testing DensityTransform')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.ctd_L2_density'
process_definition.executable['class'] = 'DensityTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
config.process.interval = 1.0
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('dens_stream_def', parameter_dictionary_id=pdict_id)
dens_stream_id, _ = self.pubsub.create_stream('test_density', stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.density = dens_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
#---------------------------------------------------------------------------------------------
# Create a subscriber that will receive the density granule from the ctd transform
#---------------------------------------------------------------------------------------------
ar_dens = gevent.event.AsyncResult()
def subscriber3(m,r,s):
ar_dens.set(m)
sub_dens = StandaloneStreamSubscriber('sub_dens', subscriber3)
self.addCleanup(sub_dens.stop)
sub_dens_id = self.pubsub.create_subscription('subscription_dens',
stream_ids=[dens_stream_id],
exchange_name='sub_dens')
self.pubsub.activate_subscription(sub_dens_id)
self.queue_cleanup.append(sub_dens.xn.queue)
sub_dens.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_dens.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('density'))
self.check_density_algorithm_execution(publish_granule, result)
def test_ctd_L2_salinity(self):
'''
Test that packets are processed by the ctd_L1_salinity transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='SalinityTransform',
description='For testing SalinityTransform')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.ctd_L2_salinity'
process_definition.executable['class'] = 'SalinityTransform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('sal_stream_def', parameter_dictionary_id=pdict_id)
sal_stream_id, _ = self.pubsub.create_stream('test_salinity', stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.salinity = sal_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
#---------------------------------------------------------------------------------------------
# Create a subscriber that will receive the salinity granule from the ctd transform
#---------------------------------------------------------------------------------------------
ar_sal = gevent.event.AsyncResult()
def subscriber3(m,r,s):
ar_sal.set(m)
sub_sal = StandaloneStreamSubscriber('sub_sal', subscriber3)
self.addCleanup(sub_sal.stop)
sub_sal_id = self.pubsub.create_subscription('subscription_sal',
stream_ids=[sal_stream_id],
exchange_name='sub_sal')
self.pubsub.activate_subscription(sub_sal_id)
self.queue_cleanup.append(sub_sal.xn.queue)
sub_sal.start()
#------------------------------------------------------------------------------------------------------
# Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
#------------------------------------------------------------------------------------------------------
# Do all the routing stuff for the publishing
routing_key = 'stream_id.stream'
stream_route = StreamRoute(self.exchange_point, routing_key)
xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
xp = self.container.ex_manager.create_xp(self.exchange_point)
xn.bind('stream_id.stream', xp)
pub = StandaloneStreamPublisher('stream_id', stream_route)
# Build a packet that can be published
self.px_ctd = SimpleCtdPublisher()
publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
# Publish the packet
pub.publish(publish_granule)
#------------------------------------------------------------------------------------------------------
# Make assertions about whether the ctd transform executed its algorithm and published the correct
# granules
#------------------------------------------------------------------------------------------------------
# Get the granule that is published by the ctd transform post processing
result = ar_sal.get(timeout=10)
self.assertTrue(isinstance(result, Granule))
rdt = RecordDictionaryTool.load_from_granule(result)
self.assertTrue(rdt.__contains__('salinity'))
self.check_salinity_algorithm_execution(publish_granule, result)
def _get_new_ctd_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
rdt['time'] = numpy.arange(self.i, self.i+length)
for field in rdt:
if isinstance(rdt._pdict.get_context(field).param_type, QuantityType):
rdt[field] = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)])
g = rdt.to_granule()
self.i+=length
return g
def test_presf_L0_splitter(self):
'''
Test that packets are processed by the ctd_L1_pressure transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='Presf L0 Splitter',
description='For testing Presf L0 Splitter')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.presf_L0_splitter'
process_definition.executable['class'] = 'PresfL0Splitter'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('pres_stream_def', parameter_dictionary_id=pdict_id)
pres_stream_id, _ = self.pubsub.create_stream('test_pressure',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.absolute_pressure = pres_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
# #---------------------------------------------------------------------------------------------
# # Create subscribers that will receive the pressure granules from
# # the ctd transform
# #---------------------------------------------------------------------------------------------
#
# ar_pres = gevent.event.AsyncResult()
# def subscriber3(m,r,s):
# ar_pres.set(m)
# sub_pres = StandaloneStreamSubscriber('sub_pres', subscriber3)
# self.addCleanup(sub_pres.stop)
#
# sub_pres_id = self.pubsub.create_subscription('subscription_pres',
# stream_ids=[pres_stream_id],
# exchange_name='sub_pres')
#
# self.pubsub.activate_subscription(sub_pres_id)
#
# self.queue_cleanup.append(sub_pres.xn.queue)
#
# sub_pres.start()
#
# #------------------------------------------------------------------------------------------------------
# # Use a StandaloneStreamPublisher to publish a packet that can be then picked up by a ctd transform
# #------------------------------------------------------------------------------------------------------
#
# # Do all the routing stuff for the publishing
# routing_key = 'stream_id.stream'
# stream_route = StreamRoute(self.exchange_point, routing_key)
#
# xn = self.container.ex_manager.create_xn_queue(self.exchange_name)
# xp = self.container.ex_manager.create_xp(self.exchange_point)
# xn.bind('stream_id.stream', xp)
#
# pub = StandaloneStreamPublisher('stream_id', stream_route)
#
# # Build a packet that can be published
# self.px_ctd = SimpleCtdPublisher()
# publish_granule = self._get_new_ctd_packet(stream_definition_id=stream_def_id, length = 5)
#
# # Publish the packet
# pub.publish(publish_granule)
#
# #------------------------------------------------------------------------------------------------------
# # Make assertions about whether the ctd transform executed its algorithm and published the correct
# # granules
# #------------------------------------------------------------------------------------------------------
#
# # Get the granule that is published by the ctd transform post processing
# result = ar_pres.get(timeout=10)
# self.assertTrue(isinstance(result, Granule))
#
# rdt = RecordDictionaryTool.load_from_granule(result)
# self.assertTrue(rdt.__contains__('pressure'))
#
# self.check_pres_algorithm_execution(publish_granule, result)
#
def test_presf_L1(self):
'''
Test that packets are processed by the ctd_L1_pressure transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='PresfL1Transform',
description='For testing PresfL1Transform')
process_definition.executable['module']= 'ion.processes.data.transforms.ctd.presf_L1'
process_definition.executable['class'] = 'PresfL1Transform'
ctd_transform_proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition('pres_stream_def', parameter_dictionary_id=pdict_id)
pres_stream_id, _ = self.pubsub.create_stream('test_pressure',
stream_definition_id=stream_def_id,
exchange_point='science_data')
config.process.publish_streams.seafloor_pressure = pres_stream_id
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=ctd_transform_proc_def_id, configuration=config)
class ExternalDatasetAgentTestBase(object):
# Agent parameters.
EDA_RESOURCE_ID = '123xyz'
EDA_NAME = 'ExampleEDA'
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Initialize test members.
"""
# log.warn('Starting the container')
# Start container.
self._start_container()
# Bring up services in a deploy file
# log.warn('Starting the rel')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a pubsub client to create streams.
# log.warn('Init a pubsub client')
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# log.warn('Init a ContainerAgentClient')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
# TODO: Finish dealing with the resources and whatnot
# TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources()
self._setup_resources()
#TG: Setup/configure the granule logger to log granules as they're published
# Create agent config.
agent_config = {
'driver_config': self.DVR_CONFIG,
'stream_config': {},
'agent': {
'resource_id': self.EDA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
log.debug('TestInstrumentAgent.setup(): starting EDA.')
self._ia_pid = self._container_client.spawn_process(
name=self.EDA_NAME,
module=self.EDA_MOD,
cls=self.EDA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
########################################
# Private "setup" functions
########################################
def _setup_resources(self):
raise NotImplementedError(
'_setup_resources must be implemented in the subclass')
def create_stream_and_logger(self, name, stream_id=''):
if not stream_id or stream_id is '':
stream_id = self._pubsub_client.create_stream(name=name,
encoding='ION R2')
pid = self._container_client.spawn_process(
name=name + '_logger',
module='ion.processes.data.stream_granule_logger',
cls='StreamGranuleLogger',
config={'process': {
'stream_id': stream_id
}})
log.info(
'Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.
format(pid, stream_id))
return stream_id
def _start_finished_event_subscriber(self):
def consume_event(*args, **kwargs):
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(
self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(
len(self._finished_events_received))
log.debug(
'Called self._async_finished_result.set({0})'.format(
len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(
event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.activate()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.deactivate()
self._finished_event_subscriber = None
########################################
# Custom assertion functions
########################################
def assertListsEqual(self, lst1, lst2):
lst1.sort()
lst2.sort()
return lst1 == lst2
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('c'))
self.assertTrue(val.has_key('t'))
self.assertTrue(val.has_key('p'))
self.assertTrue(val.has_key('time'))
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val * .01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
########################################
# Test functions
########################################
def test_acquire_data(self):
cmd = AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
self._finished_count = 3
log.info('Send an unconstrained request for data (\'new data\')')
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
log.info(
'Send a second unconstrained request for data (\'new data\'), should be rejected'
)
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
config_mods = {}
log.info(
'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1'
)
config_mods['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_1')
config_mods['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
log.info(
'Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2'
)
config_mods['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_2')
config_mods['constraints'] = self.HIST_CONSTRAINTS_2
# config={'stream_id':'second_historical','TESTING':True, 'constraints':self.HIST_CONSTRAINTS_2}
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
finished = self._async_finished_result.get(timeout=10)
self.assertEqual(finished, self._finished_count)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_acquire_data_while_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {'POLLING_INTERVAL': 5}
self._ia_client.set_param(params)
self._finished_count = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
config = get_safe(self.DVR_CONFIG, 'dh_cfg', {})
log.info(
'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1'
)
config['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_1')
config['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config])
reply = self._ia_client.execute(cmd)
self.assertNotEqual(reply.status, 660)
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {'POLLING_INTERVAL': 5}
self._ia_client.set_param(params)
self._finished_count = 3
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
# Wait for some samples to roll in.
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_observatory(self):
# Test instrument driver get and set interface.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Retrieve all resource parameters.
reply = self._ia_client.get_param('DRIVER_PARAMETER_ALL')
self.assertParamDict(reply, True)
orig_config = reply
## Retrieve a subset of resource parameters.
params = ['POLLING_INTERVAL']
reply = self._ia_client.get_param(params)
self.assertParamDict(reply)
orig_params = reply
# Set a subset of resource parameters.
new_params = {
'POLLING_INTERVAL': (orig_params['POLLING_INTERVAL'] * 2),
}
self._ia_client.set_param(new_params)
check_new_params = self._ia_client.get_param(params)
self.assertParamVals(check_new_params, new_params)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_get_set_param(self):
cmd = AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
# Get a couple parameters
retval = self._ia_client.get_param(
['POLLING_INTERVAL', 'PATCHABLE_CONFIG_KEYS'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(type(retval['POLLING_INTERVAL']), int)
self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']), list)
# Attempt to get a parameter that doesn't exist
log.debug('Try getting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.get_param,
['BAD_PARAM'])
# Set the polling_interval to a new value, then get it to make sure it set properly
self._ia_client.set_param({'POLLING_INTERVAL': 10})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 10)
# Attempt to set a parameter that doesn't exist
log.debug('Try setting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.set_param,
{'BAD_PARAM': 'bad_val'})
# Attempt to set one parameter that does exist, and one that doesn't
self.assertRaises(InstParameterError, self._ia_client.set_param, {
'POLLING_INTERVAL': 20,
'BAD_PARAM': 'bad_val'
})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 20)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_initialize(self):
# Test agent initialize command. This causes creation of driver process and transition to inactive.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_states(self):
# Test agent state transitions.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._finished_count = 1
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
gevent.sleep(5)
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._async_finished_result.get(timeout=5)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_capabilities(self):
# Test the ability to retrieve agent and resource parameter and command capabilities.
acmds = self._ia_client.get_capabilities(['AGT_CMD'])
log.debug('Agent Commands: {0}'.format(acmds))
acmds = [item[1] for item in acmds]
self.assertListsEqual(acmds, AGT_CMDS.keys())
apars = self._ia_client.get_capabilities(['AGT_PAR'])
log.debug('Agent Parameters: {0}'.format(apars))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
rcmds = self._ia_client.get_capabilities(['RES_CMD'])
log.debug('Resource Commands: {0}'.format(rcmds))
rcmds = [item[1] for item in rcmds]
self.assertListsEqual(rcmds, CMDS.keys())
rpars = self._ia_client.get_capabilities(['RES_PAR'])
log.debug('Resource Parameters: {0}'.format(rpars))
rpars = [item[1] for item in rpars]
self.assertListsEqual(rpars, PARAMS.keys())
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_errors(self):
# Test illegal behavior and replies.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# Can't go active in unitialized state.
# Status 660 is state error.
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
log.info('GO ACTIVE CMD %s', str(retval))
self.assertEquals(retval.status, 660)
# Can't command driver in this state.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertEqual(reply.status, 660)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# 404 unknown agent command.
cmd = AgentCommand(command='kiss_edward')
retval = self._ia_client.execute_agent(cmd)
self.assertEquals(retval.status, 404)
# 670 unknown driver command.
cmd = AgentCommand(command='acquire_sample_please')
retval = self._ia_client.execute(cmd)
self.assertEqual(retval.status, 670)
# 630 Parameter error.
self.assertRaises(InstParameterError, self._ia_client.get_param,
'bogus bogus')
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
