class TestIntDataProcessManagementService(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
#print 'started container'
# Establish endpoint with container
container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
#print 'got CC client'
container_client.start_rel_from_url('res/deploy/r2sa.yml')
print 'started services'
# Now create client to DataProcessManagementService
self.DPMSclient = DataProcessManagementServiceClient(node=self.container.node)
self.RRclient = ResourceRegistryServiceClient(node=self.container.node)
def test_createDataProcess(self):
# test creating a new data process
print 'Creating new data process definition'
dpd_obj = IonObject(RT.DataProcessDefinition,
name='DPD1',
description='some new dpd',
process_source='some_source_reference')
try:
dprocd_id = self.DPMSclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new data process definition: %s" %ex)
print 'new dpd_id = ', dprocd_id
print 'Creating new subscription'
s_obj = IonObject(RT.Subscription,
name='S1',
description='some new subscription')
try:
s_id, version = self.RRclient.create(s_obj)
except BadRequest as ex:
self.fail("failed to create new subscription: %s" %ex)
print 'new s_id = ', s_id
print 'Creating new data product'
dprod_obj = IonObject(RT.Subscription,
name='DProd1',
description='some new data product')
try:
dprod_id, version = self.RRclient.create(dprod_obj)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new dprod_id = ', dprod_id
print 'Creating new data process'
try:
dproc_id = self.DPMSclient.create_data_process(dprocd_id, s_id, dprod_id)
except (BadRequest, NotFound, Conflict) as ex:
self.fail("failed to create new data process definition: %s" %ex)
class VisualizationIntegrationTestHelper(IonIntegrationTestCase):
def create_ctd_input_stream_and_data_product(self, data_product_name='ctd_parsed'):
cc = self.container
assertions = self.assertTrue
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
self.vis_client = VisualizationServiceClient(node=self.container.node)
#-------------------------------
# Create CTD Parsed as the initial data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data')
log.debug('Creating new CDM data product with a stream definition')
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name=data_product_name,
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product_id = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug('new ctd_parsed_data_product_id = %s' % ctd_parsed_data_product_id)
#Only ever need one device for testing purposes.
instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='SBE37IMDevice')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
ctd_stream_id = stream_ids[0]
return ctd_stream_id, ctd_parsed_data_product_id
def create_data_product(self, dp_name = "", dp_description = ""):
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters() # this creates a ParameterDictionary object
parameter_dictionary = parameter_dictionary.dump() # this returns a python dictionary
data_prod_obj = IonObject(RT.DataProduct,
name=dp_name,
description=dp_description,
temporal_domain = tdom,
spatial_domain = sdom)
data_prod_id = self.create_data_product(data_prod_obj, stream_definition_id, parameter_dictionary)
return data_prod_id, data_prod_obj
def start_simple_input_stream_process(self, ctd_stream_id):
return self.start_input_stream_process(ctd_stream_id)
def start_sinusoidal_input_stream_process(self, ctd_stream_id):
return self.start_input_stream_process(ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher', 'SinusoidalCtdPublisher')
def start_input_stream_process(self, ctd_stream_id, module = 'ion.processes.data.ctd_stream_publisher', class_name= 'SimpleCtdPublisher'):
###
### Start the process for producing the CTD data
###
# process definition for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':module,
'class':class_name
}
ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(process_definition=producer_definition)
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':ctd_stream_id,
}
}
ctd_sim_pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration)
return ctd_sim_pid
def start_output_stream_and_listen(self, ctd_stream_id, data_product_stream_ids, message_count_per_stream=10):
cc = self.container
assertions = self.assertTrue
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name = 'workflow_test',
exchange_point = 'science_data',
name = "test workflow transformations",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, container=cc)
result = gevent.event.AsyncResult()
results = []
message_count = len(data_product_stream_ids) * message_count_per_stream
def message_received(message, headers):
# Heads
results.append(message)
if len(results) >= message_count: #Only wait for so many messages - per stream
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='workflow_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
#Start the input stream process
if ctd_stream_id is not None:
ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id)
# Assert that we have received data
assertions(result.get(timeout=30))
# stop the flow parse the messages...
if ctd_stream_id is not None:
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def validate_messages(self, results):
cc = self.container
assertions = self.assertTrue
first_salinity_values = None
for message in results:
rdt = RecordDictionaryTool.load_from_granule(message)
try:
temp = get_safe(rdt, 'temp')
# psd = PointSupplementStreamParser(stream_definition=self.ctd_stream_def, stream_granule=message)
# temp = psd.get_values('temperature')
# log.info(psd.list_field_names())
except KeyError as ke:
temp = None
if temp is not None:
assertions(isinstance(temp, numpy.ndarray))
log.info( 'temperature=' + str(numpy.nanmin(temp)))
first_salinity_values = None
else:
#psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message)
#log.info( psd.list_field_names())
# Test the handy info method for the names of fields in the stream def
#assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = get_safe(rdt, 'salinity')
#salinity = psd.get_values('salinity')
log.info( 'salinity=' + str(numpy.nanmin(salinity)))
# Check to see if salinity has values
assertions(salinity != None)
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
if first_salinity_values is None:
first_salinity_values = salinity.tolist()
else:
second_salinity_values = salinity.tolist()
assertions(len(first_salinity_values) == len(second_salinity_values))
for idx in range(0,len(first_salinity_values)):
assertions(first_salinity_values[idx]*2.0 == second_salinity_values[idx])
def validate_data_ingest_retrieve(self, dataset_id):
assertions = self.assertTrue
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
#validate that data was ingested
replay_granule = self.data_retriever.retrieve_last_granule(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
salinity = get_safe(rdt, 'salinity')
assertions(salinity != None)
#retrieve all the granules from the database and check the values
replay_granule_all = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule_all)
for k, v in rdt.iteritems():
if k == 'salinity':
for val in numpy.nditer(v):
assertions(val > 0)
def create_salinity_data_process_definition(self):
# Salinity: Data Process Definition
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='ctd_salinity')
if len(dpd) > 0:
return dpd[0]
log.debug("Create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_salinity',
description='create a salinity data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Excpetion as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='Salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(sal_stream_def_id, ctd_L2_salinity_dprocdef_id )
return ctd_L2_salinity_dprocdef_id
def create_salinity_doubler_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler')
if len(dpd) > 0:
return dpd[0]
# Salinity Doubler: Data Process Definition
log.debug("Create data process definition SalinityDoublerTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='salinity_doubler',
description='create a salinity doubler data product',
module='ion.processes.data.transforms.example_double_salinity',
class_name='SalinityDoubler',
process_source='SalinityDoubler source code here...')
try:
salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new SalinityDoubler data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(salinity_double_stream_def_id, salinity_doubler_dprocdef_id )
return salinity_doubler_dprocdef_id
def create_transform_process(self, data_process_definition_id, data_process_input_dp_id):
data_process_definition = self.rrclient.read(data_process_definition_id)
# Find the link between the output Stream Definition resource and the Data Process Definition resource
stream_ids,_ = self.rrclient.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True)
if not stream_ids:
raise Inconsistent("The data process definition %s is missing an association to an output stream definition" % data_process_definition._id )
process_output_stream_def_id = stream_ids[0]
#Concatenate the name of the workflow and data process definition for the name of the data product output
data_process_name = data_process_definition.name
# Create the output data product of the transform
transform_dp_obj = IonObject(RT.DataProduct, name=data_process_name,description=data_process_definition.description)
transform_dp_id = self.dataproductclient.create_data_product(transform_dp_obj, process_output_stream_def_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=transform_dp_id)
#last one out of the for loop is the output product id
output_data_product_id = transform_dp_id
# Create the transform data process
log.debug("create data_process and start it")
data_process_id = self.dataprocessclient.create_data_process(data_process_definition._id, [data_process_input_dp_id], {'output':transform_dp_id})
self.dataprocessclient.activate_data_process(data_process_id)
#Find the id of the output data stream
stream_ids, _ = self.rrclient.find_objects(transform_dp_id, PRED.hasStream, None, True)
if not stream_ids:
raise Inconsistent("The data process %s is missing an association to an output stream" % data_process_id )
return data_process_id, output_data_product_id
def create_google_dt_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform')
if len(dpd) > 0:
return dpd[0]
# Data Process Definition
log.debug("Create data process definition GoogleDtTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='google_dt_transform',
description='Convert data streams to Google DataTables',
module='ion.processes.data.transforms.viz.google_dt',
class_name='VizTransformGoogleDT',
process_source='VizTransformGoogleDT source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new VizTransformGoogleDT data process definition: %s" %ex)
# create a stream definition for the data from the
stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformGoogleDT.outgoing_stream_def, name='VizTransformGoogleDT')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id )
return procdef_id
def validate_google_dt_transform_results(self, results):
cc = self.container
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results,Granule):
results =[results]
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
gdt_data = get_safe(rdt, 'google_dt_components')
# IF this granule does not contains google dt, skip
if gdt_data == None:
continue
gdt = gdt_data[0]
assertions(gdt['viz_product_type'] == 'google_dt' )
assertions(len(gdt['data_description']) >= 0) # Need to come up with a better check
assertions(len(gdt['data_content']) >= 0)
def create_mpl_graphs_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform')
if len(dpd) > 0:
return dpd[0]
#Data Process Definition
log.debug("Create data process definition MatplotlibGraphsTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='mpl_graphs_transform',
description='Convert data streams to Matplotlib graphs',
module='ion.processes.data.transforms.viz.matplotlib_graphs',
class_name='VizTransformMatplotlibGraphs',
process_source='VizTransformMatplotlibGraphs source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new VizTransformMatplotlibGraphs data process definition: %s" %ex)
# create a stream definition for the data
stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformMatplotlibGraphs.outgoing_stream_def, name='VizTransformMatplotlibGraphs')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id )
return procdef_id
def validate_mpl_graphs_transform_results(self, results):
cc = self.container
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results,Granule):
results =[results]
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
graphs = get_safe(rdt, 'matplotlib_graphs')
if graphs == None:
continue
for graph in graphs[0]:
# At this point only dictionaries containing image data should be passed
# For some reason non dictionary values are filtering through.
if not isinstance(graph, dict):
continue
assertions(graph['viz_product_type'] == 'matplotlib_graphs' )
# check to see if the list (numpy array) contains actual images
assertions(imghdr.what(graph['image_name'], h = graph['image_obj']) == 'png')
def validate_vis_service_google_dt_results(self, results):
assertions = self.assertTrue
assertions(results)
gdt_str = (results.lstrip("google.visualization.Query.setResponse(")).rstrip(")")
assertions(len(gdt_str) > 0)
return
def validate_vis_service_mpl_graphs_results(self, results):
assertions = self.assertTrue
assertions(results)
# check to see if the object passed is a dictionary with a valid image object in it
image_format = results["content_type"].lstrip("image/")
assertions(imghdr.what(results['image_name'], h = base64.decodestring(results['image_obj'])) == image_format)
return
class TestCTDTransformsIntegration(IonIntegrationTestCase):
pdict_id = None
def setUp(self):
# Start container
# print 'instantiating container'
self._start_container()
# container = Container()
# print 'starting container'
# container.start()
# print 'started container'
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
print "started services"
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
def create_logger(self, name, stream_id=""):
# logger process
producer_definition = ProcessDefinition(name=name + "_logger")
producer_definition.executable = {
"module": "ion.processes.data.stream_granule_logger",
"class": "StreamGranuleLogger",
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {"process": {"stream_id": stream_id}}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id, configuration=configuration
)
return pid
def _create_instrument_model(self):
instModel_obj = IonObject(
RT.InstrumentModel,
name="SBE37IMModel",
description="SBE37IMModel",
stream_configuration={"raw": "ctd_raw_param_dict", "parsed": "ctd_parsed_param_dict"},
)
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
return instModel_id
def _create_instrument_agent(self, instModel_id):
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
)
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
return instAgent_id
def _create_instrument_device(self, instModel_id):
instDevice_obj = IonObject(
RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345"
)
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id
)
return instDevice_id
def _create_instrument_agent_instance(self, instAgent_id, instDevice_id):
port_agent_config = {
"device_addr": "sbe37-simulator.oceanobservatories.org",
"device_port": 4001,
"process_type": PortAgentProcessType.UNIX,
"binary_path": "port_agent",
"command_port": 4003,
"data_port": 4000,
"log_level": 5,
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstance",
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config,
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id
)
return instAgentInstance_id
def _create_param_dicts(self):
tdom, sdom = time_series_domain()
self.sdom = sdom.dump()
self.tdom = tdom.dump()
self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
def _create_input_data_products(self, ctd_stream_def_id, instDevice_id):
dp_obj = IonObject(
RT.DataProduct,
name="the parsed data",
description="ctd stream test",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
# ---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
# ---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
# ---------------------------------------------------------------------------
# Create CTD Raw as the second data product
# ---------------------------------------------------------------------------
if not self.pdict_id:
self._create_param_dicts()
raw_stream_def_id = self.pubsubclient.create_stream_definition(
name="SBE37_RAW", parameter_dictionary_id=self.pdict_id
)
dp_obj = IonObject(
RT.DataProduct,
name="the raw data",
description="raw stream test",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product)
# ---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
# ---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
# ---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
# ---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print "Data product streams2 = ", stream_ids
return ctd_parsed_data_product
def _create_data_process_definitions(self):
# -------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L0_all",
description="transform ctd package into three separate L0 streams",
module="ion.processes.data.transforms.ctd.ctd_L0_all",
class_name="ctd_L0_all",
)
self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L1 Conductivity: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L1_conductivity",
description="create the L1 conductivity data product",
module="ion.processes.data.transforms.ctd.ctd_L1_conductivity",
class_name="CTDL1ConductivityTransform",
)
self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L1 Pressure: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L1_pressure",
description="create the L1 pressure data product",
module="ion.processes.data.transforms.ctd.ctd_L1_pressure",
class_name="CTDL1PressureTransform",
)
self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L1 Temperature: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L1_temperature",
description="create the L1 temperature data product",
module="ion.processes.data.transforms.ctd.ctd_L1_temperature",
class_name="CTDL1TemperatureTransform",
)
self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L2 Salinity: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L2_salinity",
description="create the L1 temperature data product",
module="ion.processes.data.transforms.ctd.ctd_L2_salinity",
class_name="SalinityTransform",
)
self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L2 Density: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L2_density",
description="create the L1 temperature data product",
module="ion.processes.data.transforms.ctd.ctd_L2_density",
class_name="DensityTransform",
)
self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
return (
self.ctd_L0_all_dprocdef_id,
self.ctd_L1_conductivity_dprocdef_id,
self.ctd_L1_pressure_dprocdef_id,
self.ctd_L1_temperature_dprocdef_id,
self.ctd_L2_salinity_dprocdef_id,
self.ctd_L2_density_dprocdef_id,
)
def _create_stream_definitions(self):
if not self.pdict_id:
self._create_param_dicts()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name="L0_Conductivity", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding="conductivity"
)
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name="L0_Pressure", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding="pressure"
)
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name="L0_Temperature", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding="temperature"
)
return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id
def _create_l0_output_data_products(
self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id
):
output_products = {}
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Conductivity",
description="transform output conductivity",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id
)
output_products["conductivity"] = self.ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_conductivity_output_dp_id)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Pressure",
description="transform output pressure",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id
)
output_products["pressure"] = self.ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_pressure_output_dp_id)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Temperature",
description="transform output temperature",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id
)
output_products["temperature"] = self.ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_temperature_output_dp_id)
return output_products
def _create_l1_out_data_products(self):
ctd_l1_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L1_Conductivity",
description="transform output L1 conductivity",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_conductivity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l1_conductivity", stream_ids[0])
self.loggerpids.append(pid)
ctd_l1_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L1_Pressure",
description="transform output L1 pressure",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_pressure_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l1_pressure", stream_ids[0])
self.loggerpids.append(pid)
ctd_l1_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L1_Temperature",
description="transform output L1 temperature",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_temperature_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l1_temperature", stream_ids[0])
self.loggerpids.append(pid)
def _create_l2_out_data_products(self):
# -------------------------------
# L2 Salinity - Density: Output Data Products
# -------------------------------
if not self.pdict_id:
self._create_param_dicts()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(
name="L2_salinity", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding="salinity"
)
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(
name="L2_Density", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding="density"
)
ctd_l2_salinity_output_dp_obj = IonObject(
RT.DataProduct,
name="L2_Salinity",
description="transform output L2 salinity",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_salinity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l2_salinity", stream_ids[0])
self.loggerpids.append(pid)
ctd_l2_density_output_dp_obj = IonObject(
RT.DataProduct,
name="L2_Density",
description="transform output pressure",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_density_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l2_density", stream_ids[0])
self.loggerpids.append(pid)
def test_createTransformsThenActivateInstrument(self):
self.loggerpids = []
# -------------------------------------------------------------------------------------
# Create InstrumentModel
# -------------------------------------------------------------------------------------
instModel_id = self._create_instrument_model()
# -------------------------------------------------------------------------------------
# Create InstrumentAgent
# -------------------------------------------------------------------------------------
instAgent_id = self._create_instrument_agent(instModel_id)
# -------------------------------------------------------------------------------------
# Create InstrumentDevice
# -------------------------------------------------------------------------------------
instDevice_id = self._create_instrument_device(instModel_id)
# -------------------------------------------------------------------------------------
# Create Instrument Agent Instance
# -------------------------------------------------------------------------------------
instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id, instDevice_id)
# -------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
# -------------------------------------------------------------------------------------
self._create_param_dicts()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
name="SBE37_CDM", parameter_dictionary_id=self.pdict_id
)
# -------------------------------------------------------------------------------------
# Create two data products
# -------------------------------------------------------------------------------------
ctd_parsed_data_product = self._create_input_data_products(ctd_stream_def_id, instDevice_id)
# -------------------------------------------------------------------------------------
# Create data process definitions
# -------------------------------------------------------------------------------------
self._create_data_process_definitions()
# -------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
# -------------------------------------------------------------------------------------
outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id = (
self._create_stream_definitions()
)
self.output_products = {}
self.output_products = self._create_l0_output_data_products(
outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id
)
self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(
name="L1_conductivity", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding="conductivity"
)
self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(
name="L1_Pressure", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding="pressure"
)
self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(
name="L1_Temperature", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding="temperature"
)
self._create_l1_out_data_products()
self._create_l2_out_data_products()
# -------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
# -------------------------------------------------------------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products
)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
# -------------------------------------------------------------------------------------
# L1 Conductivity: Create the data process
# -------------------------------------------------------------------------------------
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_conductivity_dprocdef_id,
[self.ctd_l0_conductivity_output_dp_id],
{"conductivity": self.ctd_l1_conductivity_output_dp_id},
)
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
# -------------------------------------------------------------------------------------
# L1 Pressure: Create the data process
# -------------------------------------------------------------------------------------
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_pressure_dprocdef_id,
[self.ctd_l0_pressure_output_dp_id],
{"pressure": self.ctd_l1_pressure_output_dp_id},
)
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
# -------------------------------------------------------------------------------------
# L1 Temperature: Create the data process
# -------------------------------------------------------------------------------------
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_temperature_dprocdef_id,
[self.ctd_l0_temperature_output_dp_id],
{"temperature": self.ctd_l1_temperature_output_dp_id},
)
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
# -------------------------------------------------------------------------------------
# L2 Salinity: Create the data process
# -------------------------------------------------------------------------------------
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {"salinity": self.ctd_l2_salinity_output_dp_id}
)
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
# -------------------------------------------------------------------------------------
# L2 Density: Create the data process
# -------------------------------------------------------------------------------------
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {"density": self.ctd_l2_density_output_dp_id}
)
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
# -------------------------------------------------------------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
# -------------------------------------------------------------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(
self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id
)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
# Wait for instrument agent to spawn
gate = ProcessStateGate(
self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING
)
self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds")
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()
)
# -------------------------------------------------------------------------------------
# Streaming
# -------------------------------------------------------------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state))
self.assertTrue(state, "DRIVER_STATE_COMMAND")
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
# todo ResourceAgentClient no longer has method set_param
# # Make sure the sampling rate and transmission are sane.
# params = {
# SBE37Parameter.NAVG : 1,
# SBE37Parameter.INTERVAL : 5,
# SBE37Parameter.TXREALTIME : True
# }
# self._ia_client.set_param(params)
# todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it?
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
gevent.sleep(15)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it?
# log.debug("test_activateInstrumentStream: calling go_observatory")
# cmd = AgentCommand(command='go_observatory')
# reply = self._ia_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
# -------------------------------------------------------------------------------------------------
# Cleanup processes
# -------------------------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# create missing data process definition
dpd_obj = IonObject(RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dataprocessclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip('not ready')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [parsed_config] )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes) )
self.assertEqual(3, len(extended_product.process_input_data_products) )
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestCTDTransformsIntegration(IonIntegrationTestCase):
pdict_id = None
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def _create_instrument_model(self):
instModel_obj = IonObject( RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
return instModel_id
def _create_instrument_agent(self, instModel_id):
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict')
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
return instAgent_id
def _create_instrument_device(self, instModel_id):
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
return instDevice_id
def _create_instrument_agent_instance(self, instAgent_id,instDevice_id):
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
return instAgentInstance_id
def _create_param_dicts(self):
tdom, sdom = time_series_domain()
self.sdom = sdom.dump()
self.tdom = tdom.dump()
self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
def _create_input_data_products(self, ctd_stream_def_id, instDevice_id, ):
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
#---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
#---------------------------------------------------------------------------
# Create CTD Raw as the second data product
#---------------------------------------------------------------------------
if not self.pdict_id:
self._create_param_dicts()
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=self.pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product)
#---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
#---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
#---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
#---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
return ctd_parsed_data_product
def _create_data_process_definitions(self):
#-------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------------------------------------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------------------------------------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------------------------------------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------------------------------------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------------------------------------------------------------
# L2 Density: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
return self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id,\
self.ctd_L1_pressure_dprocdef_id,self.ctd_L1_temperature_dprocdef_id, \
self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id
def _create_stream_definitions(self):
if not self.pdict_id:
self._create_param_dicts()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding= 'pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding='temperature' )
return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id
def _create_l0_output_data_products(self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id):
out_data_prods = []
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
out_data_prods.append(self.ctd_l0_conductivity_output_dp_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_conductivity_output_dp_id)
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
out_data_prods.append(self.ctd_l0_pressure_output_dp_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_pressure_output_dp_id)
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
out_data_prods.append(self.ctd_l0_temperature_output_dp_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_temperature_output_dp_id)
return out_data_prods
def _create_l1_out_data_products(self):
ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
self.outgoing_stream_l1_conductivity_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_conductivity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] )
self.loggerpids.append(pid)
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj,
self.outgoing_stream_l1_pressure_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_pressure_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_pressure', stream_ids[0] )
self.loggerpids.append(pid)
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj,
self.outgoing_stream_l1_temperature_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_temperature_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_temperature', stream_ids[0] )
self.loggerpids.append(pid)
def _create_l2_out_data_products(self):
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
if not self.pdict_id: self._create_param_dicts()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding='density' )
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_salinity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_salinity', stream_ids[0] )
self.loggerpids.append(pid)
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
temporal_domain = self.tdom,
spatial_domain = self.sdom)
self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_density_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_density', stream_ids[0] )
self.loggerpids.append(pid)
@unittest.skip('This test errors on coi-nightly, may be OBE.')
def test_createTransformsThenActivateInstrument(self):
self.loggerpids = []
#-------------------------------------------------------------------------------------
# Create InstrumentModel
#-------------------------------------------------------------------------------------
instModel_id = self._create_instrument_model()
#-------------------------------------------------------------------------------------
# Create InstrumentAgent
#-------------------------------------------------------------------------------------
instAgent_id = self._create_instrument_agent(instModel_id)
#-------------------------------------------------------------------------------------
# Create InstrumentDevice
#-------------------------------------------------------------------------------------
instDevice_id = self._create_instrument_device(instModel_id)
#-------------------------------------------------------------------------------------
# Create Instrument Agent Instance
#-------------------------------------------------------------------------------------
instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id,instDevice_id )
#-------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#-------------------------------------------------------------------------------------
self._create_param_dicts()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_CDM',
parameter_dictionary_id=self.pdict_id)
#-------------------------------------------------------------------------------------
# Create two data products
#-------------------------------------------------------------------------------------
ctd_parsed_data_product = self._create_input_data_products(ctd_stream_def_id,instDevice_id)
#-------------------------------------------------------------------------------------
# Create data process definitions
#-------------------------------------------------------------------------------------
self._create_data_process_definitions()
#-------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------------------------------------------------------------
outgoing_stream_l0_conductivity_id, \
outgoing_stream_l0_pressure_id, \
outgoing_stream_l0_temperature_id = self._create_stream_definitions()
self.out_prod_ids = self._create_l0_output_data_products(outgoing_stream_l0_conductivity_id,outgoing_stream_l0_pressure_id,outgoing_stream_l0_temperature_id)
self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding='pressure' )
self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding= 'temperature' )
self._create_l1_out_data_products()
self._create_l2_out_data_products()
#-------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------------------------------------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = self.ctd_L0_all_dprocdef_id,
in_data_product_ids = [ctd_parsed_data_product],
out_data_product_ids = self.out_prod_ids)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
data_process = self.rrclient.read(ctd_l0_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcess, id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,process_ids[0])
extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id)
self.assertEquals(extended_process.computed.operational_state.status, ComputedValueAvailability.NOTAVAILABLE)
self.assertEquals(data_process.message_controllable, True)
#-------------------------------------------------------------------------------------
# L1 Conductivity: Create the data process
#-------------------------------------------------------------------------------------
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_conductivity_dprocdef_id, [self.ctd_l0_conductivity_output_dp_id], [self.ctd_l1_conductivity_output_dp_id])
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
data_process = self.rrclient.read(l1_conductivity_data_process_id)
process_ids, _ = self.rrclient.find_objects(subject=l1_conductivity_data_process_id, predicate=PRED.hasProcess, id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,process_ids[0])
#-------------------------------------------------------------------------------------
# L1 Pressure: Create the data process
#-------------------------------------------------------------------------------------
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_pressure_dprocdef_id, [self.ctd_l0_pressure_output_dp_id], [self.ctd_l1_pressure_output_dp_id])
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
data_process = self.rrclient.read(l1_pressure_data_process_id)
process_ids, _ = self.rrclient.find_objects(subject=l1_pressure_data_process_id, predicate=PRED.hasProcess, id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0])
#-------------------------------------------------------------------------------------
# L1 Temperature: Create the data process
#-------------------------------------------------------------------------------------
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L1_temperature_dprocdef_id, [self.ctd_l0_temperature_output_dp_id], [self.ctd_l1_temperature_output_dp_id])
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
data_process = self.rrclient.read(l1_temperature_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(subject=l1_temperature_all_data_process_id, predicate=PRED.hasProcess, id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0])
#-------------------------------------------------------------------------------------
# L2 Salinity: Create the data process
#-------------------------------------------------------------------------------------
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_salinity_output_dp_id])
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
data_process = self.rrclient.read(l2_salinity_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(subject=l2_salinity_all_data_process_id, predicate=PRED.hasProcess, id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0])
#-------------------------------------------------------------------------------------
# L2 Density: Create the data process
#-------------------------------------------------------------------------------------
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], [self.ctd_l2_density_output_dp_id])
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
data_process = self.rrclient.read(l2_density_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(subject=l2_density_all_data_process_id, predicate=PRED.hasProcess, id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0])
#-------------------------------------------------------------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------------------------------------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
# Wait for instrument agent to spawn
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds")
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id, to_name=gate.process_id, process=FakeProcess())
#-------------------------------------------------------------------------------------
# Streaming
#-------------------------------------------------------------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state))
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
#todo ResourceAgentClient no longer has method set_param
# # Make sure the sampling rate and transmission are sane.
# params = {
# SBE37Parameter.NAVG : 1,
# SBE37Parameter.INTERVAL : 5,
# SBE37Parameter.TXREALTIME : True
# }
# self._ia_client.set_param(params)
#todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it?
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# This gevent sleep is there to test the autosample time, which will show something different from default
# only if the instrument runs for over a minute
gevent.sleep(90)
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
autosample_string = extended_instrument.computed.uptime.value
autosampling_time = int(autosample_string.split()[4])
self.assertTrue(autosampling_time > 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
#todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it?
# log.debug("test_activateInstrumentStream: calling go_observatory")
# cmd = AgentCommand(command='go_observatory')
# reply = self._ia_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
#-------------------------------------------------------------------------------------------------
# Cleanup processes
#-------------------------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
#--------------------------------------------------------------------------------
# Cleanup data products
#--------------------------------------------------------------------------------
dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct, id_only=True)
for dp_id in dp_ids:
self.dataproductclient.delete_data_product(dp_id)
class TestTransformWorker(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process=TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process = process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client, self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def push_granule(self, data_product_id):
'''
Publishes and monitors that the granule arrived
'''
datasets, _ = self.rrclient.find_objects(data_product_id, PRED.hasDataset, id_only=True)
dataset_monitor = DatasetMonitor(datasets[0])
rdt = self.ph.rdt_for_data_product(data_product_id)
self.ph.fill_parsed_rdt(rdt)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
assert dataset_monitor.wait()
dataset_monitor.stop()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_transform_worker(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.dp_list = []
self.data_process_objs = []
self._output_stream_ids = []
self.granule_verified = Event()
self.worker_assigned_event_verified = Event()
self.dp_created_event_verified = Event()
self.heartbeat_event_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
self.start_event_listener()
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process()
self.dp_list.append(dataprocess_id)
# validate the repository for data product algorithms persists the new resources NEW SA-1
# create_data_process call created one of each
dpd_ids, _ = self.rrclient.find_resources(restype=OT.DataProcessDefinition, id_only=False)
# there will be more than one becuase of the DPDs that reperesent the PFs in the data product above
self.assertTrue(dpd_ids is not None)
dp_ids, _ = self.rrclient.find_resources(restype=OT.DataProcess, id_only=False)
# only one DP becuase the PFs that are in the code dataproduct above are not activated yet.
self.assertEquals(len(dp_ids), 1)
# validate the name and version label NEW SA - 2
dataprocessdef_obj = self.dataprocessclient.read_data_process_definition(dataprocessdef_id)
self.assertEqual(dataprocessdef_obj.version_label, '1.0a')
self.assertEqual(dataprocessdef_obj.name, 'add_arrays')
# validate that the DPD has an attachment NEW SA - 21
attachment_ids, assoc_ids = self.rrclient.find_objects(dataprocessdef_id, PRED.hasAttachment, RT.Attachment, True)
self.assertEqual(len(attachment_ids), 1)
attachment_obj = self.rrclient.read_attachment(attachment_ids[0])
log.debug('attachment: %s', attachment_obj)
# validate that the data process resource has input and output data products associated
# L4-CI-SA-RQ-364 and NEW SA-3
outproduct_ids, assoc_ids = self.rrclient.find_objects(dataprocess_id, PRED.hasOutputProduct, RT.DataProduct, True)
self.assertEqual(len(outproduct_ids), 1)
inproduct_ids, assoc_ids = self.rrclient.find_objects(dataprocess_id, PRED.hasInputProduct, RT.DataProduct, True)
self.assertEqual(len(inproduct_ids), 1)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id,_ = self.rrclient.find_objects(subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 2)
self.assertTrue(self.input_dp_id in output_data_product_provenance[output_data_product_id[0]]['parents'])
self.assertTrue(output_data_product_provenance[output_data_product_id[0]]['parents'][self.input_dp_id]['data_process_definition_id'] == dataprocessdef_id)
# NEW SA - 4 | Data processing shall include the appropriate data product algorithm name and version number in
# the metadata of each output data product created by the data product algorithm.
output_data_product_obj,_ = self.rrclient.find_objects(subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=False)
self.assertTrue(output_data_product_obj[0].name != None)
self.assertTrue(output_data_product_obj[0]._rev != None)
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(subject=dataprocess_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False)
log.debug('test_transform_worker subscription_obj: %s', subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(name='parsed_subscription', stream_ids=[self.stream_id], exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id, stream_route=stream_route )
for n in range(1, 101):
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
# validate that the output granule is received and the updated value is correct
self.assertTrue(self.granule_verified.wait(self.wait_time))
# validate that the data process loaded into worker event is received (L4-CI-SA-RQ-182)
self.assertTrue(self.worker_assigned_event_verified.wait(self.wait_time))
# validate that the data process create (with data product ids) event is received (NEW SA -42)
self.assertTrue(self.dp_created_event_verified.wait(self.wait_time))
# validate that the data process heartbeat event is received (for every hundred granules processed) (L4-CI-SA-RQ-182)
#this takes a while so set wait limit to large value
self.assertTrue(self.heartbeat_event_verified.wait(200))
# validate that the code from the transform function can be retrieve via inspect_data_process_definition
src = self.dataprocessclient.inspect_data_process_definition(dataprocessdef_id)
self.assertIn( 'def add_arrays(a, b)', src)
# now delete the DPD and DP then verify that the resources are retired so that information required for provenance are still available
self.dataprocessclient.delete_data_process(dataprocess_id)
self.dataprocessclient.delete_data_process_definition(dataprocessdef_id)
in_dp_objs, _ = self.rrclient.find_objects(subject=dataprocess_id, predicate=PRED.hasInputProduct, object_type=RT.DataProduct, id_only=True)
self.assertTrue(in_dp_objs is not None)
dpd_objs, _ = self.rrclient.find_subjects(subject_type=RT.DataProcessDefinition, predicate=PRED.hasDataProcess, object=dataprocess_id, id_only=True)
self.assertTrue(dpd_objs is not None)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_instrumentdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='test_ctd_device')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice, name='test_ctd_device', description="test_ctd_device", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process()
self.addCleanup(self.dataprocessclient.delete_data_process, dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition, dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id,_ = self.rrclient.find_objects(subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[output_data_product_id[0]]['parents'])
self.assertTrue(instDevice_id in output_data_product_provenance[self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[instDevice_id]['type'] == 'InstrumentDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_platformdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
platform_device_obj,_ = self.rrclient.find_resources(restype=RT.PlatformDevice, name='TestPlatform')
if platform_device_obj:
platform_device_id = platform_device_obj[0]._id
else:
platform_device_obj = IonObject(RT.PlatformDevice, name='TestPlatform', description="TestPlatform", serial_number="12345" )
platform_device_id = self.imsclient.create_platform_device(platform_device=platform_device_obj)
self.damsclient.assign_data_product(input_resource_id=platform_device_id, data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process()
self.addCleanup(self.dataprocessclient.delete_data_process, dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition, dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id,_ = self.rrclient.find_objects(subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[output_data_product_id[0]]['parents'])
self.assertTrue(platform_device_id in output_data_product_provenance[self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[platform_device_id]['type'] == 'PlatformDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_event_transform_worker(self):
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# test that a data process (type: data-product-in / event-out) can be defined and launched.
# verify that event fields are correctly populated
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
# create the DPD and two DPs
self.event_data_process_id = self.create_event_data_processes()
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(subject=self.event_data_process_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False)
log.debug('test_event_transform_worker subscription_obj: %s', subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(name='parsed_subscription', stream_ids=[self.stream_id], exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id, stream_route=stream_route )
self.start_event_transform_listener()
self.data_modified = Event()
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
self.assertTrue(self.event_verified.wait(self.wait_time))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_bad_argument_map(self):
self._output_stream_ids = []
# test that a data process (type: data-product-in / data-product-out) parameter mapping it validated during
# data process creation and that the correct exception is raised for both input and output.
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product', description='input test stream',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
# Set up DPD and DP #2 - array add function
tf_obj = IonObject(RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri='http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='add_array_func' )
# create the data process with invalid argument map
argument_map = {"arr1": "foo", "arr2": "bar"}
output_param = "salinity"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(ex.message, "Input data product does not contain the parameters defined in argument map")
# create the data process with invalid output parameter name
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "foo"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(ex.message, "Output data product does not contain the output parameter name provided")
def create_event_data_processes(self):
# two data processes using one transform and one DPD
argument_map= {"a": "salinity"}
# set up DPD and DP #2 - array add function
tf_obj = IonObject(RT.TransformFunction,
name='validate_salinity_array',
description='validate_salinity_array',
function='validate_salinity_array',
module="ion.processes.data.transforms.test.test_transform_worker",
arguments=['a'],
function_type=TransformFunctionType.TRANSFORM
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(RT.DataProcessDefinition,
name='validate_salinity_array',
description='validate_salinity_array',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='validate_salinity_array' )
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id],
outputs=None, argument_map=argument_map)
self.damsclient.register_process(dp1_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id)
return dp1_data_process_id
def create_data_process(self):
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "salinity"
# set up DPD and DP #2 - array add function
tf_obj = IonObject(RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri='http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
version_label='1.0a'
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='add_array_func' )
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(data_process_definition_id=add_array_dpd_id, inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id], argument_map=argument_map, out_param_name=output_param)
self.damsclient.register_process(dp1_data_process_id)
#self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id)
# add an attachment object to this DPD to test new SA-21
import msgpack
attachment_content = 'foo bar'
attachment_obj = IonObject( RT.Attachment,
name='test_attachment',
attachment_type=AttachmentType.ASCII,
content_type='text/plain',
content=msgpack.packb(attachment_content))
att_id = self.rrclient.create_attachment(add_array_dpd_id, attachment_obj)
self.addCleanup(self.rrclient.delete_attachment, att_id)
return add_array_dpd_id, dp1_data_process_id, dp1_func_output_dp_id
def create_output_data_product(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject( RT.DataProduct,
name='data_process1_data_product',
description='output of add array func',
temporal_domain = self.time_dom.dump(),
spatial_domain = self.spatial_dom.dump())
dp1_func_output_dp_id = self.dataproductclient.create_data_product(dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product, dp1_func_output_dp_id)
# retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id, PRED.hasStream, None, True)
self._output_stream_ids.append(stream_ids[0])
subscription_id = self.pubsub_client.create_subscription('validator', data_product_ids=[dp1_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription, subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s', stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
self.granule_verified.set()
validator = StandaloneStreamSubscriber('validator', callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, subscription_id)
return dp1_func_output_dp_id
def validate_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
data_process_event = args[0]
log.debug("DataProcessStatusEvent: %s" , str(data_process_event.__dict__))
# if data process already created, check origin
if self.dp_list:
self.assertIn( data_process_event.origin, self.dp_list)
# if this is a heartbeat event then 100 granules have been processed
if 'data process status update.' in data_process_event.description:
self.heartbeat_event_verified.set()
else:
# else check that this is the assign event
if 'Data process assigned to transform worker' in data_process_event.description:
self.worker_assigned_event_verified.set()
elif 'Data process created for data product' in data_process_event.description:
self.dp_created_event_verified.set()
def validate_output_granule(self, msg, route, stream_id):
self.assertIn( stream_id, self._output_stream_ids)
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule rdt: %s', rdt)
sal_val = rdt['salinity']
np.testing.assert_array_equal(sal_val, np.array([3]))
def start_event_listener(self):
es = EventSubscriber(event_type=OT.DataProcessStatusEvent, callback=self.validate_event)
es.start()
self.addCleanup(es.stop)
def validate_transform_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
status_alert_event = args[0]
np.testing.assert_array_equal(status_alert_event.origin, self.stream_id )
np.testing.assert_array_equal(status_alert_event.values, np.array([self.event_data_process_id]))
log.debug("DeviceStatusAlertEvent: %s" , str(status_alert_event.__dict__))
self.event_verified.set()
def start_event_transform_listener(self):
es = EventSubscriber(event_type=OT.DeviceStatusAlertEvent, callback=self.validate_transform_event)
es.start()
self.addCleanup(es.stop)
def test_download(self):
egg_url = 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
egg_path = TransformWorker.download_egg(egg_url)
import pkg_resources
pkg_resources.working_set.add_entry(egg_path)
from ion_example.add_arrays import add_arrays
a = add_arrays(1,2)
self.assertEquals(a,3)
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def test_get_data_product_provenance_report(self):
#Create a test device
device_obj = Device(name='Device1',
description='test instrument site')
device_id, _ = self.rrclient.create(device_obj)
self.addCleanup(self.rrclient.delete, device_id)
#Create a test DataProduct
data_product1_obj = DataProduct(name='DataProduct1',
description='test data product 1')
data_product1_id, _ = self.rrclient.create(data_product1_obj)
self.addCleanup(self.rrclient.delete, data_product1_id)
#Create a test DataProcess
data_process_obj = DataProcess(name='DataProcess',
description='test data process')
data_process_id, _ = self.rrclient.create(data_process_obj)
self.addCleanup(self.rrclient.delete, data_process_id)
#Create a second test DataProduct
data_product2_obj = DataProduct(name='DataProduct2',
description='test data product 2')
data_product2_id, _ = self.rrclient.create(data_product2_obj)
self.addCleanup(self.rrclient.delete, data_product2_id)
#Create a test DataProducer
data_producer_obj = DataProducer(name='DataProducer',
description='test data producer')
data_producer_id, rev = self.rrclient.create(data_producer_obj)
#Link the DataProcess to the second DataProduct manually
assoc_id, _ = self.rrclient.create_association(subject=data_process_id, predicate=PRED.hasInputProduct, object=data_product2_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
# Register the instrument and process. This links the device and the data process
# with their own producers
self.damsclient.register_instrument(device_id)
self.addCleanup(self.damsclient.unregister_instrument, device_id)
self.damsclient.register_process(data_process_id)
self.addCleanup(self.damsclient.unregister_process, data_process_id)
#Manually link the first DataProduct with the test DataProducer
assoc_id, _ = self.rrclient.create_association(subject=data_product1_id, predicate=PRED.hasDataProducer, object=data_producer_id)
#Get the DataProducer linked to the DataProcess (created in register_process above)
#Associate that with with DataProduct1's DataProducer
data_process_producer_ids, _ = self.rrclient.find_objects(subject=data_process_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True)
assoc_id, _ = self.rrclient.create_association(subject=data_process_producer_ids[0], predicate=PRED.hasParent, object=data_producer_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
#Get the DataProducer linked to the Device (created in register_instrument
#Associate that with the DataProcess's DataProducer
device_producer_ids, _ = self.rrclient.find_objects(subject=device_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True)
assoc_id, _ = self.rrclient.create_association(subject=data_producer_id, predicate=PRED.hasParent, object=device_producer_ids[0])
#Create the links between the Device, DataProducts, DataProcess, and all DataProducers
self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=data_product1_id)
self.addCleanup(self.damsclient.unassign_data_product, device_id, data_product1_id)
self.damsclient.assign_data_product(input_resource_id=data_process_id, data_product_id=data_product2_id)
self.addCleanup(self.damsclient.unassign_data_product, data_process_id, data_product2_id)
#Traverse through the relationships to get the links between objects
res = self.dpmsclient.get_data_product_provenance_report(data_product2_id)
#Make sure there are four keys
self.assertEqual(len(res.keys()), 4)
parent_count = 0
config_count = 0
for v in res.itervalues():
if 'parent' in v:
parent_count += 1
if 'config' in v:
config_count += 1
#Make sure there are three parents and four configs
self.assertEqual(parent_count, 3)
self.assertEqual(config_count, 4)
@unittest.skip('This test is obsolete with new framework')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict' )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [parsed_config] )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
input_data_products = [ctd_parsed_data_product]
output_data_products = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id]
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L0_all_dprocdef_id,
in_data_product_ids = input_data_products,
out_data_product_ids = output_data_products
)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_conductivity_dprocdef_id,
in_data_product_ids = [ctd_l0_conductivity_output_dp_id],
out_data_product_ids = [ctd_l1_conductivity_output_dp_id])
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_pressure_dprocdef_id,
in_data_product_ids = [ctd_l0_pressure_output_dp_id],
out_data_product_ids = [ctd_l1_pressure_output_dp_id])
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_temperature_dprocdef_id,
in_data_product_ids = [ctd_l0_temperature_output_dp_id],
out_data_product_ids = [ctd_l1_temperature_output_dp_id])
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_salinity_dprocdef_id,
in_data_product_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id],
out_data_product_ids = [ctd_l2_salinity_output_dp_id])
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
in_dp_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id]
out_dp_ids = [ctd_l2_density_output_dp_id]
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_density_dprocdef_id,
in_data_product_ids = in_dp_ids,
out_data_product_ids = out_dp_ids)
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes) )
self.assertEqual(3, len(extended_product.process_input_data_products) )
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
print results
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
# @unittest.skip('not working')
def test_createDataProcess(self):
#-------------------------------
# Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new data process definition: %s" %ex)
# test Data Process Definition creation in rr
dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id)
self.assertEquals(dprocdef_obj.name,'ctd_L0_all')
# Create an input instrument
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
# Register the instrument so that the data producer and stream object are created
data_producer_id = self.damsclient.register_instrument(instrument_id)
log.debug("TestIntDataProcessMgmtServiceMultiOut data_producer_id %s" % data_producer_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = ctd_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data')
self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id )
#-------------------------------
# Input Data Product
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create input data product")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
input_dp_obj = IonObject( RT.DataProduct,
name='InputDataProduct',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproductclient.create_data_product(input_dp_obj, ctd_stream_def_id, parameter_dictionary)
self.damsclient.assign_data_product(instrument_id, input_dp_id)
# Retrieve the stream via the DataProduct->Stream associations
stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True)
log.debug("TestIntDataProcessMgmtServiceMultiOut: in stream_ids " + str(stream_ids))
self.in_stream_id = stream_ids[0]
log.debug("TestIntDataProcessMgmtServiceMultiOut: Input Stream: " + str( self.in_stream_id))
#-------------------------------
# Output Data Product
#-------------------------------
outgoing_stream_conductivity = L0_conductivity_stream_definition()
outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_conductivity, name='conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id )
outgoing_stream_pressure = L0_pressure_stream_definition()
outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_pressure, name='pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id )
outgoing_stream_temperature = L0_temperature_stream_definition()
outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_temperature, name='temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id )
self.output_products={}
log.debug("TestIntDataProcessMgmtServiceMultiOut: create output data product conductivity")
output_dp_obj = IonObject(RT.DataProduct,
name='conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id, parameter_dictionary)
self.output_products['conductivity'] = output_dp_id_1
self.dataproductclient.activate_data_product_persistence(data_product_id=output_dp_id_1)
log.debug("TestIntDataProcessMgmtServiceMultiOut: create output data product pressure")
output_dp_obj = IonObject(RT.DataProduct,
name='pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id, parameter_dictionary)
self.output_products['pressure'] = output_dp_id_2
self.dataproductclient.activate_data_product_persistence(data_product_id=output_dp_id_2)
log.debug("TestIntDataProcessMgmtServiceMultiOut: create output data product temperature")
output_dp_obj = IonObject(RT.DataProduct,
name='temperature',
description='transform output ',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id, parameter_dictionary)
self.output_products['temperature'] = output_dp_id_3
self.dataproductclient.activate_data_product_persistence(data_product_id=output_dp_id_3)
#-------------------------------
# Create the data process
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create_data_process start")
try:
dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("TestIntDataProcessMgmtServiceMultiOut: create_data_process return")
# these assigns happen inside create_data_process
#self.damsclient.assign_data_product(input_resource_id=dproc_id, data_product_id=output_dp_id_1)
#self.damsclient.assign_data_product(input_resource_id=dproc_id, data_product_id=output_dp_id_2)
#self.damsclient.assign_data_product(input_resource_id=dproc_id, data_product_id=output_dp_id_3)
# @unittest.skip('not working.. Fix activate_data_product_persistence()')
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}})
def test_createDataProcessUsingSim(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'test_createDataProcessUsingSim: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'test_createDataProcessUsingSim: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_createDataProcessUsingSim: new InstrumentDevice id = ', instDevice_id
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", svr_addr="localhost",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver",
cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
comms_server_address="localhost", comms_server_port=8888)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'test_createDataProcessUsingSim: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
print 'test_createDataProcessUsingSim: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'test_createDataProcessUsingSim: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='ctd_raw',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("test_createDataProcessUsingSim: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_createDataProcessUsingSim: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id,
parameter_dictionary)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_createDataProcessUsingSim: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id,
parameter_dictionary)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_createDataProcessUsingSim: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id,
parameter_dictionary)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# Create listener for data process events and verify that events are received.
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("test_createDataProcessUsingSim: create data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createDataProcessUsingSim: data_process created: %s", str(ctd_l0_all_data_process_id))
#-------------------------------
# Retrieve a list of all data process defintions in RR and validate that the DPD is listed
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions
log.debug("test_createDataProcessUsingSim: activate_data_process ")
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
#todo: check that activate event is received L4-CI-SA-RQ-367
# todo: monitor process to se eif it is active (sa-rq-182)
# todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call
log.debug("test_createDataProcessUsingSim: call CEI interface to monitor ")
self.dataproductclient.suspend_data_product_persistence(data_product_id=ctd_raw_data_product)
self.dataproductclient.suspend_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
self.dataproductclient.suspend_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
self.dataproductclient.suspend_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
log.debug("test_createDataProcessUsingSim: deactivate_data_process ")
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
# self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
print "started services"
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
def cleanupprocs(self):
stm = os.popen("ps -e | grep ion.agents.port.logger_process")
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
stm = os.popen("ps -e | grep ion.agents.instrument.zmq_driver_process")
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
stm = os.popen("rm /tmp/*.pid.txt")
@unittest.skip("timeout on start inst agent?")
def test_reassignPrimaryDevice(self):
# ensure no processes or pids are left around by agents or Sims
self.cleanupprocs()
# Set up the preconditions
# Set up the preconditions
# ingestion configuration parameters
self.exchange_point_id = "science_data"
self.number_of_workers = 2
self.hdf_storage = HdfStorage(relative_path="ingest")
self.couch_storage = CouchStorage(datastore_name="test_datastore")
self.XP = "science_data"
self.exchange_name = "ingestion_queue"
# -------------------------------
# Create ingestion configuration and activate it
# -------------------------------
ingestion_configuration_id = self.ingestclient.create_ingestion_configuration(
exchange_point_id=self.exchange_point_id,
couch_storage=self.couch_storage,
hdf_storage=self.hdf_storage,
number_of_workers=self.number_of_workers,
)
print "test_deployAsPrimaryDevice: ingestion_configuration_id", ingestion_configuration_id
# activate an ingestion configuration
ret = self.ingestclient.activate_ingestion_configuration(ingestion_configuration_id)
# -------------------------------
# Create InstrumentModel
# -------------------------------
instModel_obj = IonObject(
RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model_label="SBE37IMModel"
)
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
print "test_deployAsPrimaryDevice: new InstrumentModel id = ", instModel_id
# -------------------------------
# Create InstrumentAgent
# -------------------------------
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
print "test_deployAsPrimaryDevice: new InstrumentAgent id = ", instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# -------------------------------
# Create Instrument Site
# -------------------------------
instrumentSite_obj = IonObject(RT.InstrumentSite, name="instrumentSite1", description="SBE37IMInstrumentSite")
try:
instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id="")
except BadRequest as ex:
self.fail("failed to create new InstrumentSite: %s" % ex)
print "test_deployAsPrimaryDevice: new instrumentSite id = ", instrumentSite_id
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id)
# -------------------------------
# Create Old InstrumentDevice
# -------------------------------
instDevice_obj = IonObject(
RT.InstrumentDevice,
name="SBE37IMDeviceYear1",
description="SBE37IMDevice for the FIRST year of deployment",
serial_number="12345",
)
try:
oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print "test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ", oldInstDevice_id
# deploy this device to the logical slot
self.omsclient.deploy_instrument_device_to_instrument_site(oldInstDevice_id, instrumentSite_id)
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE)
# set this device as the current primary device
self.omsclient.deploy_as_primary_instrument_device_to_instrument_site(oldInstDevice_id, instrumentSite_id)
# -------------------------------
# Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information
# cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
# -------------------------------
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstanceYear1",
description="SBE37IMAgentInstance Year 1",
svr_addr="localhost",
driver_module="ion.agents.instrument.drivers.sbe37.sbe37_driver",
driver_class="SBE37Driver",
cmd_port=5556,
evt_port=5557,
comms_method="ethernet",
comms_device_address="localhost",
comms_device_port=4001,
comms_server_address="localhost",
comms_server_port=8888,
)
oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, oldInstDevice_id
)
# -------------------------------
# Create CTD Parsed as the Year 1 data product
# -------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print "test_deployAsPrimaryDevice: new Stream Definition id = ", ctd_stream_def_id
print "Creating new CDM data product with a stream definition"
dp_obj = IonObject(RT.DataProduct, name="ctd_parsed_year1", description="ctd stream test year 1")
try:
ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year1
self.damsclient.assign_data_product(
input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1
)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True)
print "test_deployAsPrimaryDevice: Data product streams1 = ", stream_ids
# -------------------------------
# Create New InstrumentDevice
# -------------------------------
instDevice_obj_2 = IonObject(
RT.InstrumentDevice,
name="SBE37IMDeviceYear2",
description="SBE37IMDevice for the SECOND year of deployment",
serial_number="67890",
)
try:
newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print "test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ", newInstDevice_id
# deploy this device to the logical slot
self.omsclient.deploy_instrument_device_to_instrument_site(newInstDevice_id, instrumentSite_id)
# set the LCSTATE
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE)
instDevice_obj_2 = self.rrclient.read(newInstDevice_id)
log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate))
# -------------------------------
# Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information
# -------------------------------
instAgentInstance_new__obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstanceYear2",
description="SBE37IMAgentInstance Year 2",
svr_addr="localhost",
driver_module="ion.agents.instrument.drivers.sbe37.sbe37_driver",
driver_class="SBE37Driver",
cmd_port=5556,
evt_port=5557,
comms_method="ethernet",
comms_device_address="localhost",
comms_device_port=4002,
comms_server_address="localhost",
comms_server_port=8888,
)
newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_new__obj, instAgent_id, newInstDevice_id
)
# -------------------------------
# Create CTD Parsed as the Year 2 data product
# -------------------------------
# create a stream definition for the data from the ctd simulator
# ctd_stream_def = SBE37_CDM_stream_definition()
# ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print "test_deployAsPrimaryDevice: new Stream Definition id = ", ctd_stream_def_id
print "Creating new CDM data product with a stream definition"
dp_obj = IonObject(RT.DataProduct, name="ctd_parsed_year2", description="ctd stream test year 2")
try:
ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year2
self.damsclient.assign_data_product(
input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2
)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True)
print "test_deployAsPrimaryDevice: Data product streams2 = ", stream_ids
# -------------------------------
# Logical Data Product: Data Process Definition
# -------------------------------
# log.debug(" test_deployAsPrimaryDevice: create data process definition logical_transform")
# dpd_obj = IonObject(RT.DataProcessDefinition,
# name='logical_transform',
# description='send the packet from the in stream to the out stream unchanged',
# module='ion.processes.data.transforms.logical_transform',
# class_name='logical_transform',
# process_source='some_source_reference')
# try:
# logical_transform_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# except BadRequest as ex:
# self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
# -------------------------------
log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L0_all",
description="transform ctd package into three separate L0 streams",
module="ion.processes.data.transforms.ctd.ctd_L0_all",
class_name="ctd_L0_all",
process_source="some_source_reference",
)
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" % ex)
# -------------------------------
# Logical Transform: Output Data Products
# -------------------------------
# outgoing_logical_stream_def = SBE37_CDM_stream_definition()
# outgoing_logical_stream_def_id = self.pubsubclient.create_stream_definition(container=outgoing_logical_stream_def)
# self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_logical_stream_def_id, logical_transform_dprocdef_id )
#
# log.debug("test_deployAsPrimaryDevice: create output parsed data product for Logical Instrument")
# ctd_logical_output_dp_obj = IonObject(RT.DataProduct, name='ctd_parsed_logical',description='ctd parsed from the logical instrument')
# instrument_site_output_dp_id = self.dataproductclient.create_data_product(ctd_logical_output_dp_obj, outgoing_logical_stream_def_id)
# self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id, persist_data=True, persist_metadata=True)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
# -------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_conductivity, name="L0_Conductivity"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id
)
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_pressure, name="L0_Pressure"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id
)
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_temperature, name="L0_Temperature"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id
)
self.output_products = {}
log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct, name="L0_Conductivity", description="transform output conductivity"
)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id
)
self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id
# self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct, name="L0_Pressure", description="transform output pressure"
)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id
)
self.output_products["pressure"] = ctd_l0_pressure_output_dp_id
# self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct, name="L0_Temperature", description="transform output temperature"
)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id
)
self.output_products["temperature"] = ctd_l0_temperature_output_dp_id
# self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True)
# -------------------------------
# CTD Logical: Create the data process
# -------------------------------
# log.debug("test_deployAsPrimaryDevice: create ctd_parsed logical data_process start")
# try:
# ctd_parsed_logical_data_process_id = self.dataprocessclient.create_data_process(logical_transform_dprocdef_id, ctd_parsed_data_product_year1, {'output':instrument_site_output_dp_id})
# self.dataprocessclient.activate_data_process(ctd_parsed_logical_data_process_id)
# except BadRequest as ex:
# self.fail("failed to create new data process: %s" %ex)
# log.debug("test_deployAsPrimaryDevice: create L0 all data_process return")
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product)
# -------------------------------
log.debug("test_deployAsPrimaryDevice: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products
)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug("test_deployAsPrimaryDevice: create L0 all data_process return")
# -------------------------------
# Launch InstrumentAgentInstance Sim1, connect to the resource agent client
# -------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id)
inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id)
print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent1_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim1 = ResourceAgentClient(
"iaclient Sim1", name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess()
)
print "activate_instrument: got _ia_client_sim1 %s", self._ia_client_sim1
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1))
# -------------------------------
# Launch InstrumentAgentInstance Sim2, connect to the resource agent client
# -------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id)
print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent2_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim2 = ResourceAgentClient(
"iaclient Sim2", name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess()
)
print "activate_instrument: got _ia_client_sim2 %s", self._ia_client_sim2
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2))
# -------------------------------
# Streaming Sim1 (old instrument)
# -------------------------------
cmd = AgentCommand(command="initialize")
retval = self._ia_client_sim1.execute_agent(cmd)
print retval
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 initialize %s", str(retval))
time.sleep(2)
cmd = AgentCommand(command="go_active")
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command="run")
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrument: calling go_streaming ")
cmd = AgentCommand(command="go_streaming")
reply = self._ia_client_sim1.execute(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_streaming %s", str(reply))
# -------------------------------
# Streaming Sim 2 (new instrument)
# -------------------------------
cmd = AgentCommand(command="initialize")
retval = self._ia_client_sim2.execute_agent(cmd)
print retval
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 initialize %s", str(retval))
time.sleep(2)
cmd = AgentCommand(command="go_active")
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command="run")
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrument: calling go_streaming ")
cmd = AgentCommand(command="go_streaming")
reply = self._ia_client_sim2.execute(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_streaming %s", str(reply))
time.sleep(20)
# -------------------------------
# Shutdown Sim1 (old instrument)
# -------------------------------
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command="go_observatory")
reply = self._ia_client_sim1.execute(cmd)
log.debug("test_activateInstrument: _ia_client_sim1 return from go_observatory %s", str(reply))
time.sleep(5)
log.debug("test_deployAsPrimaryDevice:: calling go_inactive ")
cmd = AgentCommand(command="go_inactive")
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from go_inactive %s", str(reply))
time.sleep(2)
log.debug("test_deployAsPrimaryDevice:: calling reset ")
cmd = AgentCommand(command="reset")
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply))
time.sleep(2)
# -------------------------------
# Shutdown Sim2 (old instrument)
# -------------------------------
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command="go_observatory")
reply = self._ia_client_sim2.execute(cmd)
log.debug("test_activateInstrument: _ia_client_sim2 return from go_observatory %s", str(reply))
time.sleep(8)
log.debug("test_deployAsPrimaryDevice:: calling go_inactive ")
cmd = AgentCommand(command="go_inactive")
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from go_inactive %s", str(reply))
time.sleep(2)
log.debug("test_deployAsPrimaryDevice:: calling reset ")
cmd = AgentCommand(command="reset")
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply))
time.sleep(2)
log.debug("test_deployAsPrimaryDevice:: calling reset ")
cmd = AgentCommand(command="reset")
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from reset %s", str(reply))
time.sleep(2)
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id)
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
#self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
def cleanupprocs(self):
stm = os.popen('ps -e | grep ion.agents.port.logger_process')
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid,signal.SIGKILL)
stm = os.popen('ps -e | grep ion.agents.instrument.zmq_driver_process')
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid,signal.SIGKILL)
# stm = os.popen('rm /tmp/*.pid.txt')
@unittest.skip ("Deprecated by IngestionManagement refactor, timeout on start inst agent?")
def test_deploy_activate_full(self):
# ensure no processes or pids are left around by agents or Sims
self.cleanupprocs()
# Set up the preconditions
# Set up the preconditions
# ingestion configuration parameters
exchange_point_id = 'science_data'
exchange_name = 'ingestion_queue'
#-------------------------------
# Create ingestion configuration and activate it
#-------------------------------
q = IngestionQueue()
ingestion_configuration_id = self.ingestclient.create_ingestion_configuration(
name=exchange_name,
exchange_point_id=exchange_point_id,
queues=[q])
print 'test_deployAsPrimaryDevice: ingestion_configuration_id', ingestion_configuration_id
# activate an ingestion configuration
# = self.ingestclient.activate_ingestion_configuration(ingestion_configuration_id)
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'test_deployAsPrimaryDevice: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'test_deployAsPrimaryDevice: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create Instrument Site
#-------------------------------
instrumentSite_obj = IonObject(RT.InstrumentSite, name='instrumentSite1', description="SBE37IMInstrumentSite" )
try:
instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id='')
except BadRequest as ex:
self.fail("failed to create new InstrumentSite: %s" %ex)
print 'test_deployAsPrimaryDevice: new instrumentSite id = ', instrumentSite_id
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id)
#-------------------------------
# Logical Transform: Output Data Products
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
log.debug("test_deployAsPrimaryDevice: create output parsed data product for Logical Instrument")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
ctd_logical_output_dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_logical',
description='ctd parsed from the logical instrument',
temporal_domain = tdom,
spatial_domain = sdom)
instrument_site_output_dp_id = self.dataproductclient.create_data_product(ctd_logical_output_dp_obj,
ctd_stream_def_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id)
self.omsclient.create_site_data_product(instrumentSite_id, instrument_site_output_dp_id)
#-------------------------------
# Create Old InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDeviceYear1',
description="SBE37IMDevice for the FIRST year of deployment",
serial_number="12345" )
try:
oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ', oldInstDevice_id
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE)
#-------------------------------
# Create Old Deployment
#-------------------------------
deployment_obj = IonObject(RT.Deployment, name='first deployment')
oldDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id)
#-------------------------------
# Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information
# cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
#-------------------------------
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstanceYear1',
description="SBE37IMAgentInstance Year 1",
svr_addr="localhost",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
cmd_port=5556,
evt_port=5557,
comms_method="ethernet",
comms_device_address="localhost",
comms_device_port=4001,
comms_server_address="localhost",
comms_server_port=8888)
oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
oldInstDevice_id)
#-------------------------------
# Create CTD Parsed as the Year 1 data product and attach to instrument
#-------------------------------
print 'test_deployAsPrimaryDevice: new Stream Definition id = ', ctd_stream_def_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_year1',
description='ctd stream test year 1',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year1
self.damsclient.assign_data_product(input_resource_id=oldInstDevice_id,
data_product_id=ctd_parsed_data_product_year1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True)
print 'test_deployAsPrimaryDevice: Data product streams1 = ', stream_ids
#-------------------------------
# Create New InstrumentDevice
#-------------------------------
instDevice_obj_2 = IonObject(RT.InstrumentDevice,
name='SBE37IMDeviceYear2',
description="SBE37IMDevice for the SECOND year of deployment",
serial_number="67890" )
try:
newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ', newInstDevice_id
#set the LCSTATE
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE)
instDevice_obj_2 = self.rrclient.read(newInstDevice_id)
log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ",
str(instDevice_obj_2.lcstate))
#-------------------------------
# Create Old Deployment
#-------------------------------
deployment_obj = IonObject(RT.Deployment, name='second deployment')
newDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id)
#-------------------------------
# Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information
#-------------------------------
instAgentInstance_new__obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstanceYear2',
description="SBE37IMAgentInstance Year 2",
svr_addr="localhost",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
cmd_port=5556, evt_port=5557,
comms_method="ethernet",
comms_device_address="localhost",
comms_device_port=4002,
comms_server_address="localhost",
comms_server_port=8888)
newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_new__obj,
instAgent_id,
newInstDevice_id)
#-------------------------------
# Create CTD Parsed as the Year 2 data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
# ctd_stream_def = SBE37_CDM_stream_definition()
# ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'test_deployAsPrimaryDevice: new Stream Definition id = ', ctd_stream_def_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_year2',
description='ctd stream test year 2',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year2
self.damsclient.assign_data_product(input_resource_id=newInstDevice_id,
data_product_id=ctd_parsed_data_product_year2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True)
print 'test_deployAsPrimaryDevice: Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id,
parameter_dictionary)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id,
parameter_dictionary )
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product( ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id,
parameter_dictionary)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product)
#-------------------------------
log.debug("test_deployAsPrimaryDevice: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_deployAsPrimaryDevice: create L0 all data_process return")
#--------------------------------
# Activate the deployment
#--------------------------------
self.omsclient.activate_deployment(oldDeployment_id)
#-------------------------------
# Launch InstrumentAgentInstance Sim1, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id)
inst_agent1_instance_obj= self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id)
print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent1_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim1 = ResourceAgentClient('iaclient Sim1', name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess())
print 'activate_instrument: got _ia_client_sim1 %s', self._ia_client_sim1
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1))
#-------------------------------
# Launch InstrumentAgentInstance Sim2, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
inst_agent2_instance_obj= self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id)
print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent2_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim2 = ResourceAgentClient('iaclient Sim2', name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess())
print 'activate_instrument: got _ia_client_sim2 %s', self._ia_client_sim2
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2))
#-------------------------------
# Streaming Sim1 (old instrument)
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client_sim1.execute_agent(cmd)
print retval
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 initialize %s", str(retval))
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command='run')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrument: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client_sim1.execute(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 go_streaming %s", str(reply))
#-------------------------------
# Streaming Sim 2 (new instrument)
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client_sim2.execute_agent(cmd)
print retval
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 initialize %s", str(retval))
time.sleep(2)
cmd = AgentCommand(command='go_active')
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command='run')
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrument: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client_sim2.execute(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 go_streaming %s", str(reply))
time.sleep(20)
#-------------------------------
# Shutdown Sim1 (old instrument)
#-------------------------------
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client_sim1.execute(cmd)
log.debug("test_activateInstrument: _ia_client_sim1 return from go_observatory %s", str(reply))
time.sleep(5)
log.debug("test_deployAsPrimaryDevice:: calling go_inactive ")
cmd = AgentCommand(command='go_inactive')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from go_inactive %s", str(reply))
time.sleep(2)
log.debug("test_deployAsPrimaryDevice:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply))
time.sleep(2)
#-------------------------------
# Shutdown Sim2 (old instrument)
#-------------------------------
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client_sim2.execute(cmd)
log.debug("test_activateInstrument: _ia_client_sim2 return from go_observatory %s", str(reply))
time.sleep(8)
log.debug("test_deployAsPrimaryDevice:: calling go_inactive ")
cmd = AgentCommand(command='go_inactive')
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from go_inactive %s", str(reply))
time.sleep(2)
log.debug("test_deployAsPrimaryDevice:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim1 return from reset %s", str(reply))
time.sleep(2)
log.debug("test_deployAsPrimaryDevice:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice:: _ia_client_sim2 return from reset %s", str(reply))
time.sleep(2)
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id)
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
def test_createDataProcess(self):
#---------------------------------------------------------------------------
# Data Process Definition
#---------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# Make assertion on the newly registered data process definition
data_process_definition = self.rrclient.read(dprocdef_id)
self.assertEquals(data_process_definition.name, 'ctd_L0_all')
self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams')
self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all')
self.assertEquals(data_process_definition.class_name, 'ctd_L0_all')
# Read the data process definition using data process management and make assertions
dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id)
self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all')
self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all')
#---------------------------------------------------------------------------
# Create an input instrument
#---------------------------------------------------------------------------
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
# Register the instrument so that the data producer and stream object are created
data_producer_id = self.damsclient.register_instrument(instrument_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id )
# Assert that the link between the stream definition and the data process definition was done
assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True)
self.assertIsNotNone(assocs)
#---------------------------------------------------------------------------
# Input Data Product
#---------------------------------------------------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_dp_obj = IonObject( RT.DataProduct,
name='InputDataProduct',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test')
#Make assertions on the input data product created
input_dp_obj = self.rrclient.read(input_dp_id)
self.assertEquals(input_dp_obj.name, 'InputDataProduct')
self.assertEquals(input_dp_obj.description, 'some new dp')
self.damsclient.assign_data_product(instrument_id, input_dp_id)
# Retrieve the stream via the DataProduct->Stream associations
stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True)
self.in_stream_id = stream_ids[0]
#---------------------------------------------------------------------------
# Output Data Product
#---------------------------------------------------------------------------
outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' )
outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' )
outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' )
self.output_products={}
output_dp_obj = IonObject(RT.DataProduct,
name='conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id)
self.output_products['conductivity'] = output_dp_id_1
output_dp_obj = IonObject(RT.DataProduct,
name='pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id)
self.output_products['pressure'] = output_dp_id_2
output_dp_obj = IonObject(RT.DataProduct,
name='temperature',
description='transform output ',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id)
self.output_products['temperature'] = output_dp_id_3
#---------------------------------------------------------------------------
# Create the data process
#---------------------------------------------------------------------------
def _create_data_process():
dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products)
return dproc_id
dproc_id = _create_data_process()
# Make assertions on the data process created
data_process = self.dataprocessclient.read_data_process(dproc_id)
# Assert that the data process has a process id attached
self.assertIsNotNone(data_process.process_id)
# Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute
self.assertIsNotNone(data_process.input_subscription_id)
output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True)
self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3]))
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}})
def test_createDataProcessUsingSim(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" )
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg")
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4002,
'data_port': 4003,
'log_level': 5,
}
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id)
dp_obj.name = 'ctd_raw'
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# Create listener for data process events and verify that events are received.
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events
#todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
data_process = self.rrclient.read(ctd_l0_all_data_process_id)
process_id = data_process.process_id
self.addCleanup(self.process_dispatcher.cancel_process, process_id)
#-------------------------------
# Wait until the process launched in the create_data_process() method is actually running, before proceeding further in this test
#-------------------------------
gate = ProcessStateGate(self.process_dispatcher.read_process, process_id, ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The data process (%s) did not spawn in 30 seconds" % process_id)
#-------------------------------
# Retrieve a list of all data process defintions in RR and validate that the DPD is listed
#-------------------------------
# todo: Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions
# todo: data topics are being handled by pub sub at the level of streams
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
#todo: check that activate event is received L4-CI-SA-RQ-367
#todo... (it looks like no event is being published when the data process is activated... so below, we just check for now
# todo... that the subscription is indeed activated) (Swarbhanu)
# todo: monitor process to see if it is active (sa-rq-182)
ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id)
input_subscription_id = ctd_l0_all_data_process.input_subscription_id
subs = self.rrclient.read(input_subscription_id)
self.assertTrue(subs.activated)
# todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the extended resources for data process definition and for data process
#-------------------------------
extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
self.assertEqual(1, len(extended_process_definition.data_processes))
log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition))
extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id)
self.assertEqual(1, len(extended_process.input_data_products))
log.debug("test_createDataProcess: extended_process %s", str(extended_process))
################################ Test the removal of data processes ##################################
#-------------------------------------------------------------------
# Test the deleting of the data process
#-------------------------------------------------------------------
# Before deleting, get the input streams, output streams and the subscriptions so that they can be checked after deleting
# dp_obj_1 = self.rrclient.read(ctd_l0_all_data_process_id)
# input_subscription_id = dp_obj_1.input_subscription_id
# out_prods, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True)
# in_prods, _ = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasInputProduct, id_only=True)
# in_streams = []
# for in_prod in in_prods:
# streams, _ = self.rrclient.find_objects(in_prod, PRED.hasStream, id_only=True)
# in_streams.extend(streams)
# out_streams = []
# for out_prod in out_prods:
# streams, _ = self.rrclient.find_objects(out_prod, PRED.hasStream, id_only=True)
# out_streams.extend(streams)
# Deleting the data process
self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id)
# Check that the data process got removed. Check the lcs state. It should be retired
dp_obj = self.rrclient.read(ctd_l0_all_data_process_id)
self.assertEquals(dp_obj.lcstate, LCS.RETIRED)
# Check for process defs still attached to the data process
dpd_assn_ids = self.rrclient.find_associations(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcessDefinition, id_only=True)
self.assertEquals(len(dpd_assn_ids), 0)
# Check for output data product still attached to the data process
out_products, assocs = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True)
self.assertEquals(len(out_products), 0)
self.assertEquals(len(assocs), 0)
# Check for input data products still attached to the data process
inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct)
self.assertEquals(len(inprod_associations), 0)
# Check for input data products still attached to the data process
inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct)
self.assertEquals(len(inprod_associations), 0)
# Check of the data process has been deactivated
self.assertIsNone(dp_obj.input_subscription_id)
# Read the original subscription id of the data process and check that it has been deactivated
with self.assertRaises(NotFound):
self.pubsubclient.read_subscription(input_subscription_id)
#-------------------------------------------------------------------
# Delete the data process definition
#-------------------------------------------------------------------
# before deleting, get the process definition being associated to in order to be able to check later if the latter gets deleted as it should
proc_def_ids, proc_def_asocs = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasProcessDefinition)
self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id)
# check that the data process definition has been retired
dp_proc_def = self.rrclient.read(ctd_L0_all_dprocdef_id)
self.assertEquals(dp_proc_def.lcstate, LCS.RETIRED)
# Check for old associations of this data process definition
proc_defs, proc_def_asocs = self.rrclient.find_objects(ctd_L0_all_dprocdef_id, PRED.hasProcessDefinition)
self.assertEquals(len(proc_defs), 0)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject= ctd_L0_all_dprocdef_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
################################ Test the removal of data processes ##################################
# Try force delete... This should simply delete the associations and the data process object
# from the resource registry
#---------------------------------------------------------------------------------------------------------------
# Force deleting a data process
#---------------------------------------------------------------------------------------------------------------
self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True)
# find all associations where this is the object
_, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
self.assertEquals(len(sbj_assns), 0)
with self.assertRaises(NotFound):
self.rrclient.read(ctd_l0_all_data_process_id)
#---------------------------------------------------------------------------------------------------------------
# Force deleting a data process definition
#---------------------------------------------------------------------------------------------------------------
self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True)
# find all associations where this is the object
_, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
self.assertEquals(len(sbj_assns), 0)
with self.assertRaises(NotFound):
self.rrclient.read(ctd_l0_all_data_process_id)
def test_transform_function_crd(self):
tf = TransformFunction(name='simple', module='pyon.ion.process', cls='SimpleProcess')
tf_id = self.dataprocessclient.create_transform_function(tf)
self.assertTrue(tf_id)
tf2_id = self.dataprocessclient.create_transform_function(tf)
self.assertEquals(tf_id, tf2_id)
tf_obj = self.dataprocessclient.read_transform_function(tf_id)
self.assertEquals([tf.name, tf.module, tf.cls, tf.function_type], [tf_obj.name, tf_obj.module, tf_obj.cls, tf_obj.function_type])
tf.module = 'dev.null'
self.assertRaises(BadRequest, self.dataprocessclient.create_transform_function, tf)
self.dataprocessclient.delete_transform_function(tf_id)
self.assertRaises(NotFound, self.dataprocessclient.read_transform_function, tf_id)
class TestOmsLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(
self._network_definition)
if log.isEnabledFor(logging.DEBUG):
log.debug("NetworkDefinition serialization:\n%s",
self._network_definition_ser)
self.platformModel_id = None
self.all_platforms = {}
self.agent_streamconfig_map = {}
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
self._set_up_DataProduct_obj()
self._set_up_PlatformModel_obj()
def _set_up_DataProduct_obj(self):
# Create data product object to be used for each of the platform log streams
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'platform_eng_parsed', id_only=True)
self.platform_eng_stream_def_id = self.pubsubcli.create_stream_definition(
name='platform_eng', parameter_dictionary_id=self.pdict_id)
self.dp_obj = IonObject(RT.DataProduct,
name='platform_eng data',
description='platform_eng test',
temporal_domain=tdom,
spatial_domain=sdom)
def _set_up_PlatformModel_obj(self):
# Create PlatformModel
platformModel_obj = IonObject(RT.PlatformModel,
name='RSNPlatformModel',
description="RSNPlatformModel")
try:
self.platformModel_id = self.imsclient.create_platform_model(
platformModel_obj)
except BadRequest as ex:
self.fail("failed to create new PLatformModel: %s" % ex)
log.debug('new PlatformModel id = %s', self.platformModel_id)
def _traverse(self, pnode, platform_id, parent_platform_objs=None):
"""
Recursive routine that repeatedly calls _prepare_platform to build
the object dictionary for each platform.
@param pnode PlatformNode
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval the dict returned by _prepare_platform at this level.
"""
log.info("Starting _traverse for %r", platform_id)
plat_objs = self._prepare_platform(pnode, platform_id,
parent_platform_objs)
self.all_platforms[platform_id] = plat_objs
# now, traverse the children:
for sub_pnode in pnode.subplatforms.itervalues():
subplatform_id = sub_pnode.platform_id
self._traverse(sub_pnode, subplatform_id, plat_objs)
return plat_objs
def _prepare_platform(self, pnode, platform_id, parent_platform_objs):
"""
This routine generalizes the manual construction originally done in
test_oms_launch.py. It is called by the recursive _traverse method so
all platforms starting from a given base platform are prepared.
Note: For simplicity in this test, sites are organized in the same
hierarchical way as the platforms themselves.
@param pnode PlatformNode
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval a dict of associated objects similar to those in
test_oms_launch
"""
site__obj = IonObject(RT.PlatformSite,
name='%s_PlatformSite' % platform_id,
description='%s_PlatformSite platform site' %
platform_id)
site_id = self.omsclient.create_platform_site(site__obj)
if parent_platform_objs:
# establish hasSite association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['site_id'],
predicate=PRED.hasSite,
object=site_id)
# prepare platform attributes and ports:
monitor_attribute_objs, monitor_attribute_dicts = self._prepare_platform_attributes(
pnode, platform_id)
port_objs, port_dicts = self._prepare_platform_ports(
pnode, platform_id)
device__obj = IonObject(
RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
# ports=port_objs,
# platform_monitor_attributes = monitor_attribute_objs
)
device__dict = dict(
ports=port_dicts,
platform_monitor_attributes=monitor_attribute_dicts)
self.device_id = self.imsclient.create_platform_device(device__obj)
self.imsclient.assign_platform_model_to_platform_device(
self.platformModel_id, self.device_id)
self.rrclient.create_association(subject=site_id,
predicate=PRED.hasDevice,
object=self.device_id)
self.damsclient.register_instrument(instrument_id=self.device_id)
if parent_platform_objs:
# establish hasDevice association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['device_id'],
predicate=PRED.hasDevice,
object=self.device_id)
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' %
platform_id)
agent_id = self.imsclient.create_platform_agent(agent__obj)
if parent_platform_objs:
# add this platform_id to parent's children:
parent_platform_objs['children'].append(platform_id)
self.imsclient.assign_platform_model_to_platform_agent(
self.platformModel_id, agent_id)
# agent_instance_obj = IonObject(RT.PlatformAgentInstance,
# name='%s_PlatformAgentInstance' % platform_id,
# description="%s_PlatformAgentInstance" % platform_id)
#
# agent_instance_id = self.imsclient.create_platform_agent_instance(
# agent_instance_obj, agent_id, device_id)
plat_objs = {
'platform_id': platform_id,
'site__obj': site__obj,
'site_id': site_id,
'device__obj': device__obj,
'device_id': self.device_id,
'agent__obj': agent__obj,
'agent_id': agent_id,
# 'agent_instance_obj': agent_instance_obj,
# 'agent_instance_id': agent_instance_id,
'children': []
}
log.info("plat_objs for platform_id %r = %s", platform_id,
str(plat_objs))
stream_config = self._create_stream_config(plat_objs)
self.agent_streamconfig_map[platform_id] = stream_config
# self.agent_streamconfig_map[platform_id] = None
# self._start_data_subscriber(agent_instance_id, stream_config)
return plat_objs
def _prepare_platform_attributes(self, pnode, platform_id):
"""
Returns the list of PlatformMonitorAttributes objects corresponding to
the attributes associated to the given platform.
"""
# TODO complete the clean-up of this method
ret_infos = dict((n, a.defn) for (n, a) in pnode.attrs.iteritems())
monitor_attribute_objs = []
monitor_attribute_dicts = []
for attrName, attrDfn in ret_infos.iteritems():
log.debug("platform_id=%r: preparing attribute=%r", platform_id,
attrName)
monitor_rate = attrDfn['monitorCycleSeconds']
units = attrDfn['units']
plat_attr_obj = IonObject(OT.PlatformMonitorAttributes,
id=attrName,
monitor_rate=monitor_rate,
units=units)
plat_attr_dict = dict(id=attrName,
monitor_rate=monitor_rate,
units=units)
monitor_attribute_objs.append(plat_attr_obj)
monitor_attribute_dicts.append(plat_attr_dict)
return monitor_attribute_objs, monitor_attribute_dicts
def _prepare_platform_ports(self, pnode, platform_id):
"""
Returns the list of PlatformPort objects corresponding to the ports
associated to the given platform.
"""
# TODO complete the clean-up of this method
port_objs = []
port_dicts = []
for port_id, network in pnode.ports.iteritems():
log.debug("platform_id=%r: preparing port=%r network=%s",
platform_id, port_id, network)
#
# Note: the name "IP" address has been changed to "network" address
# in the CI-OMS interface spec.
#
plat_port_obj = IonObject(OT.PlatformPort,
port_id=port_id,
ip_address=network)
plat_port_dict = dict(port_id=port_id, network=network)
port_objs.append(plat_port_obj)
port_dicts.append(plat_port_dict)
return port_objs, port_dicts
def _create_stream_config(self, plat_objs):
platform_id = plat_objs['platform_id']
device_id = plat_objs['device_id']
#create the log data product
self.dp_obj.name = '%s platform_eng data' % platform_id
self.data_product_id = self.dpclient.create_data_product(
data_product=self.dp_obj,
stream_definition_id=self.platform_eng_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=self.device_id,
data_product_id=self.data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(self.data_product_id,
PRED.hasStream, None, True)
stream_config = self._build_stream_config(stream_ids[0])
return stream_config
def _build_stream_config(self, stream_id=''):
platform_eng_dictionary = DatasetManagementService.get_parameter_dictionary_by_name(
'platform_eng_parsed')
#get the streamroute object from pubsub by passing the stream_id
stream_def_ids, _ = self.rrclient.find_objects(
stream_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
stream_route = self.pubsubcli.read_stream_route(stream_id=stream_id)
stream_config = {
'routing_key': stream_route.routing_key,
'stream_id': stream_id,
'stream_definition_ref': stream_def_ids[0],
'exchange_point': stream_route.exchange_point,
'parameter_dictionary': platform_eng_dictionary.dump()
}
return stream_config
def _set_platform_agent_instances(self):
"""
Once most of the objs/defs associated with all platforms are in
place, this method creates and associates the PlatformAgentInstance
elements.
"""
self.platform_configs = {}
for platform_id, plat_objs in self.all_platforms.iteritems():
PLATFORM_CONFIG = {
'platform_id': platform_id,
'agent_streamconfig_map': None, #self.agent_streamconfig_map,
'driver_config': DVR_CONFIG,
'network_definition': self._network_definition_ser
}
self.platform_configs[platform_id] = {
'platform_id': platform_id,
'agent_streamconfig_map': self.agent_streamconfig_map,
'driver_config': DVR_CONFIG,
'network_definition': self._network_definition_ser
}
agent_config = {
'platform_config': PLATFORM_CONFIG,
}
self.stream_id = self.agent_streamconfig_map[platform_id][
'stream_id']
# import pprint
# print '============== platform id within unit test: %s ===========' % platform_id
# pprint.pprint(agent_config)
#agent_config['platform_config']['agent_streamconfig_map'] = None
agent_instance_obj = IonObject(
RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=agent_config)
agent_id = plat_objs['agent_id']
device_id = plat_objs['device_id']
agent_instance_id = self.imsclient.create_platform_agent_instance(
agent_instance_obj, agent_id, self.device_id)
plat_objs['agent_instance_obj'] = agent_instance_obj
plat_objs['agent_instance_id'] = agent_instance_id
stream_config = self.agent_streamconfig_map[platform_id]
self._start_data_subscriber(agent_instance_id, stream_config)
def _start_data_subscriber(self, stream_name, stream_config):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r', stream_name)
stream_id = self.stream_id #stream_config['stream_id']
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.pubsubcli.create_subscription(name=exchange_name,
stream_ids=[stream_id])
self.pubsubcli.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(
sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self,
event_type="DeviceEvent",
sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(
sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
@skip("IMS does't net implement topology")
def test_hierarchy(self):
self._create_launch_verify(BASE_PLATFORM_ID)
@skip("Needs alignment with recent IMS changes")
def test_single_platform(self):
self._create_launch_verify('LJ01D')
def _create_launch_verify(self, base_platform_id):
# and trigger the traversal of the branch rooted at that base platform
# to create corresponding ION objects and configuration dictionaries:
pnode = self._network_definition.pnodes[base_platform_id]
base_platform_objs = self._traverse(pnode, base_platform_id)
# now that most of the topology information is there, add the
# PlatformAgentInstance elements
self._set_platform_agent_instances()
base_platform_config = self.platform_configs[base_platform_id]
log.info("base_platform_id = %r", base_platform_id)
#-------------------------------------------------------------------------------------
# Create Data Process Definition and Data Process for the eng stream monitor process
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='DemoStreamAlertTransform',
description='For testing EventTriggeredTransform_B',
module='ion.processes.data.transforms.event_alert_transform',
class_name='DemoStreamAlertTransform')
self.platform_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#THERE SHOULD BE NO STREAMDEF REQUIRED HERE.
platform_streamdef_id = self.pubsubcli.create_stream_definition(
name='platform_eng_parsed', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
platform_streamdef_id, self.platform_dprocdef_id, binding='output')
config = {
'process': {
'timer_interval': 5,
'queue_name': 'a_queue',
'variable_name': 'input_voltage',
'time_field_name': 'preferred_timestamp',
'valid_values': [-100, 100],
'timer_origin': 'Interval Timer'
}
}
platform_data_process_id = self.dataprocessclient.create_data_process(
self.platform_dprocdef_id, [self.data_product_id], {}, config)
self.dataprocessclient.activate_data_process(platform_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
platform_data_process_id)
#-------------------------------
# Launch Base Platform AgentInstance, connect to the resource agent client
#-------------------------------
agent_instance_id = base_platform_objs['agent_instance_id']
log.debug(
"about to call imsclient.start_platform_agent_instance with id=%s",
agent_instance_id)
pid = self.imsclient.start_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.imsclient.read_platform_agent_instance(
agent_instance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (90),
"The platform agent instance did not spawn in 90 seconds")
agent_instance_obj = self.imsclient.read_instrument_agent_instance(
agent_instance_id)
log.debug(
'test_oms_create_and_launch: Platform agent instance obj: %s',
str(agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient(
'paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug(" test_oms_create_and_launch:: got pa client %s",
str(self._pa_client))
log.debug("base_platform_config =\n%s", base_platform_config)
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug('Base Platform ping_agent = %s', str(retval))
# issue INITIALIZE command to the base platform, which will launch the
# creation of the whole platform hierarchy rooted at base_platform_config['platform_id']
# cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE, kwargs=dict(plat_config=base_platform_config))
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform INITIALIZE = %s', str(retval))
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_ACTIVE = %s', str(retval))
# RUN:
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RUN = %s', str(retval))
# START_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform START_MONITORING = %s', str(retval))
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(15)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
#get the extended platfrom which wil include platform aggreate status fields
extended_platform = self.imsclient.get_platform_device_extension(
self.device_id)
# log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) )
# log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) )
# log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) )
# STOP_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform STOP_MONITORING = %s', str(retval))
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_INACTIVE = %s', str(retval))
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RESET = %s', str(retval))
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.imsclient.stop_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
#@unittest.skip('not ready')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
stream_configuration= {'parsed': 'ctd_parsed_param_dict' } )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4003,
'data_port': 4000,
'log_level': 5,
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
#self.assertEqual(ComputedValueAvailability.PROVIDED, extended_product.provenance_product_list.status)
log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
#log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(1, len(extended_process_def.data_products) ) #one list because of one data process
self.assertEqual(3, len(extended_process_def.data_products[0]) ) #inside that inner list are the three output data products
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
#self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(
RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def cleanupprocs(self):
stm = os.popen('ps -e | grep ion.agents.port.logger_process')
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
stm = os.popen('ps -e | grep ion.agents.instrument.zmq_driver_process')
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
# stm = os.popen('rm /tmp/*.pid.txt')
@unittest.skip(
"Deprecated by IngestionManagement refactor, timeout on start inst agent?"
)
def test_deploy_activate_full(self):
# ensure no processes or pids are left around by agents or Sims
#self.cleanupprocs()
self.loggerpids = []
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD)
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
#-------------------------------
# Create Instrument Site
#-------------------------------
instrumentSite_obj = IonObject(RT.InstrumentSite,
name='instrumentSite1',
description="SBE37IMInstrumentSite")
try:
instrumentSite_id = self.omsclient.create_instrument_site(
instrument_site=instrumentSite_obj, parent_id='')
except BadRequest as ex:
self.fail("failed to create new InstrumentSite: %s" % ex)
print 'test_deployAsPrimaryDevice: new instrumentSite id = ', instrumentSite_id
self.omsclient.assign_instrument_model_to_instrument_site(
instModel_id, instrumentSite_id)
#-------------------------------
# Logical Transform: Output Data Products
#-------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubclient.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Old InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(
RT.InstrumentDevice,
name='SBE37IMDeviceYear1',
description="SBE37IMDevice for the FIRST year of deployment",
serial_number="12345")
try:
oldInstDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, oldInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print 'test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ', oldInstDevice_id
self.rrclient.execute_lifecycle_transition(oldInstDevice_id,
LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(oldInstDevice_id,
LCE.ENABLE)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='SiteDataProduct',
description='SiteDataProduct',
temporal_domain=tdom,
spatial_domain=sdom)
instrument_site_output_dp_id = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=oldInstDevice_id,
data_product_id=instrument_site_output_dp_id)
#self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(
instrument_site_output_dp_id, PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(
instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
#-------------------------------
# Create Old Deployment
#-------------------------------
deployment_obj = IonObject(RT.Deployment, name='first deployment')
oldDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(oldInstDevice_id,
oldDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id,
oldDeployment_id)
#-------------------------------
# Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information
# cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
raw_config = StreamConfiguration(
stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstanceYear1',
description="SBE37IMAgentInstanceYear1",
port_agent_config=port_agent_config,
stream_configurations=[raw_config, parsed_config])
oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, oldInstDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
#-------------------------------
# Create CTD Parsed as the Year 1 data product and attach to instrument
#-------------------------------
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_year1',
description='ctd stream test year 1',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year1
self.damsclient.assign_data_product(
input_resource_id=oldInstDevice_id,
data_product_id=ctd_parsed_data_product_year1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(
ctd_parsed_data_product_year1, PRED.hasStream, None, True)
print 'test_deployAsPrimaryDevice: Data product streams1 = ', stream_ids
#-------------------------------
# Create New InstrumentDevice
#-------------------------------
instDevice_obj_2 = IonObject(
RT.InstrumentDevice,
name='SBE37IMDeviceYear2',
description="SBE37IMDevice for the SECOND year of deployment",
serial_number="67890")
try:
newInstDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj_2)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, newInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print 'test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ', newInstDevice_id
#set the LCSTATE
self.rrclient.execute_lifecycle_transition(newInstDevice_id,
LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(newInstDevice_id,
LCE.ENABLE)
instDevice_obj_2 = self.rrclient.read(newInstDevice_id)
log.debug(
"test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ",
str(instDevice_obj_2.lcstate))
#-------------------------------
# Create Old Deployment
#-------------------------------
deployment_obj = IonObject(RT.Deployment, name='second deployment')
newDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(newInstDevice_id,
newDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id,
newDeployment_id)
#-------------------------------
# Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4004,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': 4005,
'data_port': 4006,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstanceYear2',
description="SBE37IMAgentInstanceYear2",
port_agent_config=port_agent_config)
newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, newInstDevice_id)
#-------------------------------
# Create CTD Parsed as the Year 2 data product
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed_year2',
description='ctd stream test year 2',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product_year2
self.damsclient.assign_data_product(
input_resource_id=newInstDevice_id,
data_product_id=ctd_parsed_data_product_year2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(
ctd_parsed_data_product_year2, PRED.hasStream, None, True)
print 'test_deployAsPrimaryDevice: Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug(
"test_deployAsPrimaryDevice: create data process definition ctd_L0_all"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new ctd_L0_all data process definition: %s" %
ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name='L0_Conductivity', parameter_dictionary_id=parsed_pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id,
ctd_L0_all_dprocdef_id,
binding='conductivity')
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name='L0_Pressure', parameter_dictionary_id=parsed_pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id,
ctd_L0_all_dprocdef_id,
binding='pressure')
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name='L0_Temperature', parameter_dictionary_id=parsed_pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id,
ctd_L0_all_dprocdef_id,
binding='temperature')
self.out_prod_dict = {}
log.debug(
"test_deployAsPrimaryDevice: create output data product L0 conductivity"
)
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_conductivity_output_dp_obj,
stream_definition_id=parsed_stream_def_id)
self.out_prod_dict['conductivity'] = ctd_l0_conductivity_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug(
"test_deployAsPrimaryDevice: create output data product L0 pressure"
)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_pressure_output_dp_obj,
stream_definition_id=parsed_stream_def_id)
self.out_prod_dict['pressure'] = ctd_l0_pressure_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug(
"test_deployAsPrimaryDevice: create output data product L0 temperature"
)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_temperature_output_dp_obj,
stream_definition_id=parsed_stream_def_id)
self.out_prod_dict['temperature'] = ctd_l0_temperature_output_dp_id
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product)
#-------------------------------
log.debug(
"test_deployAsPrimaryDevice: create L0 all data_process start")
try:
out_data_products = self.out_prod_dict.values()
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1],
out_data_products)
self.dataprocessclient.activate_data_process(
ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_deployAsPrimaryDevice: create L0 all data_process return")
#--------------------------------
# Activate the deployment
#--------------------------------
self.omsclient.activate_deployment(oldDeployment_id)
#-------------------------------
# Launch InstrumentAgentInstance Sim1, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=oldInstAgentInstance_id)
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=oldInstAgentInstance_id)
#wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(
oldInstAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
oldInstDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance(
oldInstAgentInstance_id)
print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent1_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim1 = ResourceAgentClient('iaclient Sim1',
name=gate.process_id,
process=FakeProcess())
print 'activate_instrument: got _ia_client_sim1 %s', self._ia_client_sim1
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s",
str(self._ia_client_sim1))
#-------------------------------
# Launch InstrumentAgentInstance Sim2, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=newInstAgentInstance_id)
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=newInstAgentInstance_id)
#wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(
newInstAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
oldInstDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance(
newInstAgentInstance_id)
print 'test_deployAsPrimaryDevice: Instrument agent instance obj: = ', inst_agent2_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim2 = ResourceAgentClient('iaclient Sim2',
name=gate.process_id,
process=FakeProcess())
print 'activate_instrument: got _ia_client_sim2 %s', self._ia_client_sim2
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s",
str(self._ia_client_sim2))
#-------------------------------
# Streaming Sim1 (old instrument)
#-------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval))
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: return value from go_active %s",
str(reply))
self.assertTrue(reply)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client_sim1.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: run %s", str(reply))
gevent.sleep(2)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client_sim1.execute_resource(cmd)
log.debug(
"test_activateInstrumentSample: return from START_AUTOSAMPLE: %s",
str(retval))
#-------------------------------
# Streaming Sim 2 (new instrument)
#-------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s",
str(retval))
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug(
"test_deployAsPrimaryDevice: return value from go_active_sim2 %s",
str(reply))
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client_sim2.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: run %s", str(reply))
gevent.sleep(2)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client_sim2.execute_resource(cmd)
log.debug(
"test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s",
str(retval))
gevent.sleep(10)
#-------------------------------
# Shutdown Sim1 (old instrument)
#-------------------------------
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client_sim1.execute_resource(cmd)
log.debug(
"test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
time.sleep(5)
#-------------------------------
# Shutdown Sim2 (old instrument)
#-------------------------------
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client_sim2.execute_resource(cmd)
log.debug(
"test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset_sim2 ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset_sim2 %s",
str(reply))
time.sleep(5)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
# setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
def create_logger(self, name, stream_id=""):
# logger process
producer_definition = ProcessDefinition(name=name + "_logger")
producer_definition.executable = {
"module": "ion.processes.data.stream_granule_logger",
"class": "StreamGranuleLogger",
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {"process": {"stream_id": stream_id}}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id, configuration=configuration
)
return pid
@unittest.skip("TBD")
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(
RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model="SBE37IMModel"
)
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
log.debug("new InstrumentModel id = %s ", instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug("new InstrumentAgent id = %s", instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
"test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) "
)
instDevice_obj = IonObject(
RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345"
)
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id
)
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstance",
description="SBE37IMAgentInstance",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4001,
port_agent_work_dir="/tmp/",
port_agent_delimeter=["<<", ">>"],
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id
)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug("new Stream Definition id = %s", instDevice_id)
log.debug("Creating new CDM data product with a stream definition")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(
RT.DataProduct,
name="the parsed data",
description="ctd stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("Data product streams1 = %s", stream_ids)
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
# -------------------------------
log.debug("test_activateInstrumentSample: create data process definition ctd_L0_all")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L0_all",
description="transform ctd package into three separate L0 streams",
module="ion.processes.data.transforms.ctd.ctd_L0_all",
class_name="ctd_L0_all",
process_source="some_source_reference",
)
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" % ex)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
# -------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubcli.create_stream_definition(
container=outgoing_stream_l0_conductivity, name="L0_Conductivity"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id
)
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubcli.create_stream_definition(
container=outgoing_stream_l0_pressure, name="L0_Pressure"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id
)
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubcli.create_stream_definition(
container=outgoing_stream_l0_temperature, name="L0_Temperature"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id
)
self.output_products = {}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Conductivity",
description="transform output conductivity",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id, parameter_dictionary
)
self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l0_conductivity_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l0_conductivity stream id = %s", str(stream_ids))
pid = self.create_logger(" ctd_l1_conductivity", stream_ids[0])
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Pressure",
description="transform output pressure",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary
)
self.output_products["pressure"] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l0_pressure_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l0_pressure stream id = %s", str(stream_ids))
pid = self.create_logger(" ctd_l0_pressure", stream_ids[0])
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Temperature",
description="transform output temperature",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary
)
self.output_products["temperature"] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l0_temperature_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l0_temperature stream id = %s", str(stream_ids))
pid = self.create_logger(" ctd_l0_temperature", stream_ids[0])
self.loggerpids.append(pid)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
# -------------------------------
log.debug("test_activateInstrumentSample: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [data_product_id1], self.output_products
)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
log.debug("Creating new RAW data product with a stream definition")
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(
RT.DataProduct,
name="the raw data",
description="raw stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", str(data_product_id2))
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug("Data product streams2 = %s", str(stream_ids))
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
log.debug("Instrument agent instance obj: = %s", str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client))
cmd = AgentCommand(command="initialize")
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
time.sleep(1)
log.debug("test_activateInstrumentSample: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command="go_active")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s", str(reply))
time.sleep(1)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentSample: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state),
)
cmd = AgentCommand(command="run")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
time.sleep(1)
log.debug("test_activateInstrumentSample: calling acquire_sample ")
cmd = AgentCommand(command="acquire_sample")
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentSample: return from acquire_sample %s", str(reply))
time.sleep(1)
log.debug("test_activateInstrumentSample: calling acquire_sample 2")
cmd = AgentCommand(command="acquire_sample")
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentSample: return from acquire_sample 2 %s", str(reply))
time.sleep(1)
log.debug("test_activateInstrumentSample: calling acquire_sample 3")
cmd = AgentCommand(command="acquire_sample")
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentSample: return from acquire_sample 3 %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command="reset")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s", str(reply))
time.sleep(1)
# -------------------------------
# Deactivate InstrumentAgentInstance
# -------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
@unittest.skip("TBD")
def test_activateInstrumentStream(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(
RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model="SBE37IMModel"
)
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
log.debug("new InstrumentModel id = %s ", instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug("new InstrumentAgent id = %s", instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
"test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) "
)
instDevice_obj = IonObject(
RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345"
)
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id
)
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstance",
description="SBE37IMAgentInstance",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4001,
port_agent_work_dir="/tmp/",
port_agent_delimeter=["<<", ">>"],
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id
)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug("new Stream Definition id = %s", instDevice_id)
log.debug("Creating new CDM data product with a stream definition")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(
RT.DataProduct,
name="the parsed data",
description="ctd stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("Data product streams1 = %s", stream_ids)
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
# simdata_subscription_id = self.pubsubcli.create_subscription(
# query=StreamQuery([stream_ids[0]]),
# exchange_name='Sim_data_queue',
# name='SimDataSubscription',
# description='SimData SubscriptionDescription'
# )
#
#
# def simdata_message_received(message, headers):
# input = str(message)
# log.debug("test_activateInstrumentStream: granule received: %s", input)
#
#
# subscriber_registrar = StreamSubscriberRegistrar(process=self.container, container=self.container)
# simdata_subscriber = subscriber_registrar.create_subscriber(exchange_name='Sim_data_queue', callback=simdata_message_received)
#
# # Start subscribers
# simdata_subscriber.start()
#
# # Activate subscriptions
# self.pubsubcli.activate_subscription(simdata_subscription_id)
log.debug("Creating new RAW data product with a stream definition")
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(
RT.DataProduct,
name="the raw data",
description="raw stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", str(data_product_id2))
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug("Data product streams2 = %s", str(stream_ids))
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
log.debug("test_activateInstrumentStream Instrument agent instance obj: = %s", str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess())
log.debug("test_activateInstrumentStream: got ia client %s", str(self._ia_client))
cmd = AgentCommand(command="initialize")
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: initialize %s", str(retval))
time.sleep(2)
log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command="go_active")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state),
)
cmd = AgentCommand(command="run")
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from run state: %s", str(state))
# Make sure the sampling rate and transmission are sane.
params = {SBE37Parameter.NAVG: 1, SBE37Parameter.INTERVAL: 5, SBE37Parameter.TXREALTIME: True}
self._ia_client.set_param(params)
time.sleep(2)
log.debug("test_activateInstrumentStream: calling go_streaming ")
cmd = AgentCommand(command="go_streaming")
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state))
time.sleep(5)
log.debug("test_activateInstrumentStream: calling go_observatory")
cmd = AgentCommand(command="go_observatory")
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
log.debug("test_activateInstrumentStream: calling reset ")
cmd = AgentCommand(command="reset")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result))
time.sleep(2)
# -------------------------------
# Deactivate InstrumentAgentInstance
# -------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestVisualizationWorkflowIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
def test_workflow_components(self):
cc = self.container
assertions = self.assertTrue
#-------------------------------
# Create CTD Parsed as the initial data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data')
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except Exception as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
ctd_stream_id = stream_ids[0]
###
### Setup the first transformation
###
# Salinity: Data Process Definition
log.debug("Create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_salinity',
description='create a salinity data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Excpetion as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(sal_stream_def_id, ctd_L2_salinity_dprocdef_id )
# Create the output data product of the transform
log.debug("create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity',description='transform output L2 salinity')
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, sal_stream_def_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id, persist_data=True, persist_metadata=True)
# Create the Salinity transform data process
log.debug("create L2_salinity data_process and start it")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product, {'output':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
###
### Setup the second transformation
###
# Salinity: Data Process Definition
log.debug("Create data process definition SalinityDoublerTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='salinity_doubler',
description='create a salinity doubler data product',
module='ion.processes.data.transforms.example_double_salinity',
class_name='SalinityDoubler',
process_source='SalinityDoubler source code here...')
try:
salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new SalinityDoubler data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(salinity_double_stream_def_id, salinity_doubler_dprocdef_id )
# Create the output data product of the transform
log.debug("create output data product SalinityDoubler")
salinity_doubler_output_dp_obj = IonObject(RT.DataProduct, name='SalinityDoubler',description='transform output salinity doubler')
salinity_doubler_output_dp_id = self.dataproductclient.create_data_product(salinity_doubler_output_dp_obj, salinity_double_stream_def_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=salinity_doubler_output_dp_id, persist_data=True, persist_metadata=True)
# Create the Salinity transform data process
log.debug("create L2_salinity data_process and start it")
try:
salinity_double_data_process_id = self.dataprocessclient.create_data_process(salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id, {'output':salinity_doubler_output_dp_id})
self.dataprocessclient.activate_data_process(salinity_double_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
###
### Start the process for producing the CTD data
###
# process definition for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':'ion.processes.data.ctd_stream_publisher',
'class':'SimpleCtdPublisher'
}
ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(process_definition=producer_definition)
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':ctd_stream_id,
}
}
ctd_sim_pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration)
## get the stream id for the transform outputs
stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
sal_stream_id = stream_ids[0]
stream_ids, _ = self.rrclient.find_objects(salinity_doubler_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
sal_dbl_stream_id = stream_ids[0]
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery([ctd_stream_id, sal_stream_id,sal_dbl_stream_id]),
exchange_name = 'salinity_test',
name = "test salinity subscription",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn(' data received!')
results.append(message)
if len(results) >15:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='salinity_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=20))
#Stop the transform process
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
first_salinity_values = None
for message in results:
try:
psd = PointSupplementStreamParser(stream_definition=ctd_stream_def, stream_granule=message)
temp = psd.get_values('temperature')
print psd.list_field_names()
except KeyError as ke:
temp = None
if temp is not None:
assertions(isinstance(temp, numpy.ndarray))
print 'temperature=' + str(numpy.nanmin(temp))
first_salinity_values = None
else:
psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message)
print psd.list_field_names()
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
print 'salinity=' + str(numpy.nanmin(salinity))
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
if first_salinity_values is None:
first_salinity_values = salinity.tolist()
else:
second_salinity_values = salinity.tolist()
assertions(len(first_salinity_values) == len(second_salinity_values))
for idx in range(0,len(first_salinity_values)):
assertions(first_salinity_values[idx]*2.0 == second_salinity_values[idx])
class TestTransformWorker(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process = TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(
node=self.container.node, process=process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client,
self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def push_granule(self, data_product_id):
'''
Publishes and monitors that the granule arrived
'''
datasets, _ = self.rrclient.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
dataset_monitor = DatasetMonitor(datasets[0])
rdt = self.ph.rdt_for_data_product(data_product_id)
self.ph.fill_parsed_rdt(rdt)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
assert dataset_monitor.wait()
dataset_monitor.stop()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.dp_list = []
self.data_process_objs = []
self._output_stream_ids = []
self.granule_verified = Event()
self.worker_assigned_event_verified = Event()
self.dp_created_event_verified = Event()
self.heartbeat_event_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
self.start_event_listener()
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.dp_list.append(dataprocess_id)
# validate the repository for data product algorithms persists the new resources NEW SA-1
# create_data_process call created one of each
dpd_ids, _ = self.rrclient.find_resources(
restype=OT.DataProcessDefinition, id_only=False)
# there will be more than one becuase of the DPDs that reperesent the PFs in the data product above
self.assertTrue(dpd_ids is not None)
dp_ids, _ = self.rrclient.find_resources(restype=OT.DataProcess,
id_only=False)
# only one DP becuase the PFs that are in the code dataproduct above are not activated yet.
self.assertEquals(len(dp_ids), 1)
# validate the name and version label NEW SA - 2
dataprocessdef_obj = self.dataprocessclient.read_data_process_definition(
dataprocessdef_id)
self.assertEqual(dataprocessdef_obj.version_label, '1.0a')
self.assertEqual(dataprocessdef_obj.name, 'add_arrays')
# validate that the DPD has an attachment NEW SA - 21
attachment_ids, assoc_ids = self.rrclient.find_objects(
dataprocessdef_id, PRED.hasAttachment, RT.Attachment, True)
self.assertEqual(len(attachment_ids), 1)
attachment_obj = self.rrclient.read_attachment(attachment_ids[0])
log.debug('attachment: %s', attachment_obj)
# validate that the data process resource has input and output data products associated
# L4-CI-SA-RQ-364 and NEW SA-3
outproduct_ids, assoc_ids = self.rrclient.find_objects(
dataprocess_id, PRED.hasOutputProduct, RT.DataProduct, True)
self.assertEqual(len(outproduct_ids), 1)
inproduct_ids, assoc_ids = self.rrclient.find_objects(
dataprocess_id, PRED.hasInputProduct, RT.DataProduct, True)
self.assertEqual(len(inproduct_ids), 1)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 2)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(output_data_product_provenance[
output_data_product_id[0]]['parents'][self.input_dp_id]
['data_process_definition_id'] == dataprocessdef_id)
# NEW SA - 4 | Data processing shall include the appropriate data product algorithm name and version number in
# the metadata of each output data product created by the data product algorithm.
output_data_product_obj, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=False)
self.assertTrue(output_data_product_obj[0].name != None)
self.assertTrue(output_data_product_obj[0]._rev != None)
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=dataprocess_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id,
stream_route=stream_route)
for n in range(1, 101):
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
# validate that the output granule is received and the updated value is correct
self.assertTrue(self.granule_verified.wait(self.wait_time))
# validate that the data process loaded into worker event is received (L4-CI-SA-RQ-182)
self.assertTrue(
self.worker_assigned_event_verified.wait(self.wait_time))
# validate that the data process create (with data product ids) event is received (NEW SA -42)
self.assertTrue(self.dp_created_event_verified.wait(self.wait_time))
# validate that the data process heartbeat event is received (for every hundred granules processed) (L4-CI-SA-RQ-182)
#this takes a while so set wait limit to large value
self.assertTrue(self.heartbeat_event_verified.wait(200))
# validate that the code from the transform function can be retrieve via inspect_data_process_definition
src = self.dataprocessclient.inspect_data_process_definition(
dataprocessdef_id)
self.assertIn('def add_arrays(a, b)', src)
# now delete the DPD and DP then verify that the resources are retired so that information required for provenance are still available
self.dataprocessclient.delete_data_process(dataprocess_id)
self.dataprocessclient.delete_data_process_definition(
dataprocessdef_id)
in_dp_objs, _ = self.rrclient.find_objects(
subject=dataprocess_id,
predicate=PRED.hasInputProduct,
object_type=RT.DataProduct,
id_only=True)
self.assertTrue(in_dp_objs is not None)
dpd_objs, _ = self.rrclient.find_subjects(
subject_type=RT.DataProcessDefinition,
predicate=PRED.hasDataProcess,
object=dataprocess_id,
id_only=True)
self.assertTrue(dpd_objs is not None)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_instrumentdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
instDevice_obj, _ = self.rrclient.find_resources(
restype=RT.InstrumentDevice, name='test_ctd_device')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice,
name='test_ctd_device',
description="test_ctd_device",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.addCleanup(self.dataprocessclient.delete_data_process,
dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition,
dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(instDevice_id in output_data_product_provenance[
self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[instDevice_id]['type']
== 'InstrumentDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_platformdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
platform_device_obj, _ = self.rrclient.find_resources(
restype=RT.PlatformDevice, name='TestPlatform')
if platform_device_obj:
platform_device_id = platform_device_obj[0]._id
else:
platform_device_obj = IonObject(RT.PlatformDevice,
name='TestPlatform',
description="TestPlatform",
serial_number="12345")
platform_device_id = self.imsclient.create_platform_device(
platform_device=platform_device_obj)
self.damsclient.assign_data_product(
input_resource_id=platform_device_id,
data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.addCleanup(self.dataprocessclient.delete_data_process,
dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition,
dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(platform_device_id in output_data_product_provenance[
self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[platform_device_id]
['type'] == 'PlatformDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_event_transform_worker(self):
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# test that a data process (type: data-product-in / event-out) can be defined and launched.
# verify that event fields are correctly populated
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
# create the DPD and two DPs
self.event_data_process_id = self.create_event_data_processes()
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=self.event_data_process_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_event_transform_worker subscription_obj: %s',
subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id,
stream_route=stream_route)
self.start_event_transform_listener()
self.data_modified = Event()
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
self.assertTrue(self.event_verified.wait(self.wait_time))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_bad_argument_map(self):
self._output_stream_ids = []
# test that a data process (type: data-product-in / data-product-out) parameter mapping it validated during
# data process creation and that the correct exception is raised for both input and output.
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
# Set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
# create the data process with invalid argument map
argument_map = {"arr1": "foo", "arr2": "bar"}
output_param = "salinity"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(
ex.message,
"Input data product does not contain the parameters defined in argument map"
)
# create the data process with invalid output parameter name
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "foo"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(
ex.message,
"Output data product does not contain the output parameter name provided"
)
def create_event_data_processes(self):
# two data processes using one transform and one DPD
argument_map = {"a": "salinity"}
# set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='validate_salinity_array',
description='validate_salinity_array',
function='validate_salinity_array',
module="ion.processes.data.transforms.test.test_transform_worker",
arguments=['a'],
function_type=TransformFunctionType.TRANSFORM)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='validate_salinity_array',
description='validate_salinity_array',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id,
add_array_dpd_id,
binding='validate_salinity_array')
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=None,
argument_map=argument_map)
self.damsclient.register_process(dp1_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
dp1_data_process_id)
return dp1_data_process_id
def create_data_process(self):
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "salinity"
# set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
version_label='1.0a')
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp1_data_process_id)
#self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id)
# add an attachment object to this DPD to test new SA-21
import msgpack
attachment_content = 'foo bar'
attachment_obj = IonObject(RT.Attachment,
name='test_attachment',
attachment_type=AttachmentType.ASCII,
content_type='text/plain',
content=msgpack.packb(attachment_content))
att_id = self.rrclient.create_attachment(add_array_dpd_id,
attachment_obj)
self.addCleanup(self.rrclient.delete_attachment, att_id)
return add_array_dpd_id, dp1_data_process_id, dp1_func_output_dp_id
def create_output_data_product(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject(RT.DataProduct,
name='data_process1_data_product',
description='output of add array func')
dp1_func_output_dp_id = self.dataproductclient.create_data_product(
dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp1_func_output_dp_id)
# retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_ids.append(stream_ids[0])
subscription_id = self.pubsub_client.create_subscription(
'validator', data_product_ids=[dp1_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription,
subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s',
stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
self.granule_verified.set()
validator = StandaloneStreamSubscriber('validator',
callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
subscription_id)
return dp1_func_output_dp_id
def validate_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
data_process_event = args[0]
log.debug("DataProcessStatusEvent: %s",
str(data_process_event.__dict__))
# if data process already created, check origin
if self.dp_list:
self.assertIn(data_process_event.origin, self.dp_list)
# if this is a heartbeat event then 100 granules have been processed
if 'data process status update.' in data_process_event.description:
self.heartbeat_event_verified.set()
else:
# else check that this is the assign event
if 'Data process assigned to transform worker' in data_process_event.description:
self.worker_assigned_event_verified.set()
elif 'Data process created for data product' in data_process_event.description:
self.dp_created_event_verified.set()
def validate_output_granule(self, msg, route, stream_id):
self.assertIn(stream_id, self._output_stream_ids)
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule rdt: %s', rdt)
sal_val = rdt['salinity']
np.testing.assert_array_equal(sal_val, np.array([3]))
def start_event_listener(self):
es = EventSubscriber(event_type=OT.DataProcessStatusEvent,
callback=self.validate_event)
es.start()
self.addCleanup(es.stop)
def validate_transform_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
status_alert_event = args[0]
np.testing.assert_array_equal(status_alert_event.origin,
self.stream_id)
np.testing.assert_array_equal(status_alert_event.values,
np.array([self.event_data_process_id]))
log.debug("DeviceStatusAlertEvent: %s",
str(status_alert_event.__dict__))
self.event_verified.set()
def start_event_transform_listener(self):
es = EventSubscriber(event_type=OT.DeviceStatusAlertEvent,
callback=self.validate_transform_event)
es.start()
self.addCleanup(es.stop)
def test_download(self):
egg_url = 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
egg_path = TransformWorker.download_egg(egg_url)
import pkg_resources
pkg_resources.working_set.add_entry(egg_path)
from ion_example.add_arrays import add_arrays
a = add_arrays(1, 2)
self.assertEquals(a, 3)
class TestTransformWorkerSubscriptions(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management_client = DatasetManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.time_dom, self.spatial_dom = time_series_domain()
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_multi_subscriptions(self):
self.dp_list = []
self.event1_verified = Event()
self.event2_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product_one', description='input test stream one',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_one_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product_two', description='input test stream two',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_two_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
#retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_one_id, PRED.hasStream, RT.Stream, True)
self.stream_one_id = stream_ids[0]
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_two_id, PRED.hasStream, RT.Stream, True)
self.stream_two_id = stream_ids[0]
dpd_id = self.create_data_process_definition()
dp1_func_output_dp_id, dp2_func_output_dp_id = self.create_output_data_products()
first_dp_id = self.create_data_process_one(dpd_id, dp1_func_output_dp_id)
second_dp_id = self.create_data_process_two(dpd_id, self.input_dp_two_id, dp2_func_output_dp_id)
#retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(subject=first_dp_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False)
log.debug('test_transform_worker subscription_obj: %s', subscription_objs[0])
#create subscription to stream ONE, create data process and publish granule on stream ONE
#create a queue to catch the published granules of stream ONE
self.subscription_one_id = self.pubsub_client.create_subscription(name='parsed_subscription_one', stream_ids=[self.stream_one_id], exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_one_id)
self.pubsub_client.activate_subscription(self.subscription_one_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_one_id)
stream_route_one = self.pubsub_client.read_stream_route(self.stream_one_id)
self.publisher_one = StandaloneStreamPublisher(stream_id=self.stream_one_id, stream_route=stream_route_one )
self.start_event_listener()
#data process 1 adds conductivity + pressure and puts the result in salinity
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher_one.publish(msg=rdt.to_granule(), stream_id=self.stream_one_id)
#retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(subject=second_dp_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False)
log.debug('test_transform_worker subscription_obj: %s', subscription_objs[0])
#create subscription to stream ONE and TWO, move TW subscription, create data process and publish granule on stream TWO
#create a queue to catch the published granules of stream TWO
self.subscription_two_id = self.pubsub_client.create_subscription(name='parsed_subscription_one_two', stream_ids=[self.stream_two_id], exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_two_id)
self.pubsub_client.activate_subscription(self.subscription_two_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_two_id)
stream_route_two = self.pubsub_client.read_stream_route(self.stream_two_id)
self.publisher_two = StandaloneStreamPublisher(stream_id=self.stream_two_id, stream_route=stream_route_two )
#data process 1 adds conductivity + pressure and puts the result in salinity
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher_one.publish(msg=rdt.to_granule(), stream_id=self.stream_one_id)
#data process 2 adds salinity + pressure and puts the result in conductivity
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [22]
rdt['pressure'] = [4]
rdt['salinity'] = [1]
self.publisher_two.publish(msg=rdt.to_granule(), stream_id=self.stream_two_id)
self.assertTrue(self.event2_verified.wait(self.wait_time))
self.assertTrue(self.event1_verified.wait(self.wait_time))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_two_transforms_inline(self):
self.dp_list = []
self.event1_verified = Event()
self.event2_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject( RT.DataProduct, name='input_data_product_one', description='input test stream one',
temporal_domain = self.time_dom.dump(), spatial_domain = self.spatial_dom.dump())
self.input_dp_one_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
dpd_id = self.create_data_process_definition()
dp1_func_output_dp_id, dp2_func_output_dp_id = self.create_output_data_products()
first_dp_id = self.create_data_process_one(dpd_id, dp1_func_output_dp_id)
second_dp_id = self.create_data_process_two(dpd_id, dp1_func_output_dp_id, dp2_func_output_dp_id)
#retrieve subscription from data process one
subscription_objs, _ = self.rrclient.find_objects(subject=first_dp_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False)
log.debug('test_transform_worker subscription_obj: %s', subscription_objs[0])
#retrieve the Stream for these data product
stream_ids, assoc_ids = self.rrclient.find_objects(self.input_dp_one_id, PRED.hasStream, RT.Stream, True)
self.stream_one_id = stream_ids[0]
#the input to data process two is the output from data process one
stream_ids, assoc_ids = self.rrclient.find_objects(dp1_func_output_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_two_id = stream_ids[0]
# Run provenance on the output dataproduct of the second data process to see all the links
# are as expected
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(dp2_func_output_dp_id)
# Do a basic check to see if there were 2 entries in the provenance graph. Parent and Child.
self.assertTrue(len(output_data_product_provenance) == 3)
# confirm that the linking from the output dataproduct to input dataproduct is correct
self.assertTrue(dp1_func_output_dp_id in output_data_product_provenance[dp2_func_output_dp_id]['parents'])
self.assertTrue(self.input_dp_one_id in output_data_product_provenance[dp1_func_output_dp_id]['parents'])
#create subscription to stream ONE, create data process and publish granule on stream ONE
#create a queue to catch the published granules of stream ONE
subscription_id = self.pubsub_client.create_subscription(name='parsed_subscription', stream_ids=[self.stream_one_id, self.stream_two_id], exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription, subscription_id)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, subscription_id)
stream_route_one = self.pubsub_client.read_stream_route(self.stream_one_id)
self.publisher_one = StandaloneStreamPublisher(stream_id=self.stream_one_id, stream_route=stream_route_one )
#retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(subject=second_dp_id, predicate=PRED.hasSubscription, object_type=RT.Subscription, id_only=False)
log.debug('test_transform_worker subscription_obj: %s', subscription_objs[0])
#data process 1 adds conductivity + pressure and puts the result in salinity
#data process 2 adds salinity + pressure and puts the result in conductivity
self.start_event_listener()
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher_one.publish(msg=rdt.to_granule(), stream_id=self.stream_one_id)
self.assertTrue(self.event2_verified.wait(self.wait_time))
self.assertTrue(self.event1_verified.wait(self.wait_time))
def create_data_process_definition(self):
#two data processes using one transform and one DPD
# Set up DPD and DP #2 - array add function
tf_obj = IonObject(RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri='http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(self.stream_def_id, add_array_dpd_id, binding='add_array_func' )
return add_array_dpd_id
def create_data_process_one(self, data_process_definition_id, output_dataproduct):
# Create the data process
#data process 1 adds conductivity + pressure and puts the result in salinity
argument_map = {"arr1":"conductivity", "arr2":"pressure"}
output_param = "salinity"
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=data_process_definition_id,
inputs=[self.input_dp_one_id],
outputs=[output_dataproduct],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp1_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id)
self.dp_list.append(dp1_data_process_id)
return dp1_data_process_id
def create_data_process_two(self, data_process_definition_id, input_dataproduct, output_dataproduct):
# Create the data process
#data process 2 adds salinity + pressure and puts the result in conductivity
argument_map = {'arr1':'salinity', 'arr2':'pressure'}
output_param = 'conductivity'
dp2_func_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=data_process_definition_id,
inputs=[input_dataproduct],
outputs=[output_dataproduct],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp2_func_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process, dp2_func_data_process_id)
self.dp_list.append(dp2_func_data_process_id)
return dp2_func_data_process_id
def create_output_data_products(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject( RT.DataProduct,
name='data_process1_data_product',
description='output of add array func',
temporal_domain = self.time_dom.dump(),
spatial_domain = self.spatial_dom.dump())
dp1_func_output_dp_id = self.dataproductclient.create_data_product(dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product, dp1_func_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id, PRED.hasStream, None, True)
self._output_stream_one_id = stream_ids[0]
dp2_func_outgoing_stream_id = self.pubsub_client.create_stream_definition(name='dp2_stream', parameter_dictionary_id=self.parameter_dict_id)
dp2_func_output_dp_obj = IonObject( RT.DataProduct,
name='data_process2_data_product',
description='output of add array func',
temporal_domain = self.time_dom.dump(),
spatial_domain = self.spatial_dom.dump())
dp2_func_output_dp_id = self.dataproductclient.create_data_product(dp2_func_output_dp_obj, dp2_func_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product, dp2_func_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp2_func_output_dp_id, PRED.hasStream, None, True)
self._output_stream_two_id = stream_ids[0]
subscription_id = self.pubsub_client.create_subscription('validator', data_product_ids=[dp1_func_output_dp_id, dp2_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription, subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s', stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
validator = StandaloneStreamSubscriber('validator', callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, subscription_id)
return dp1_func_output_dp_id, dp2_func_output_dp_id
def validate_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
data_process_event = args[0]
log.debug("DataProcessStatusEvent: %s" , str(data_process_event.__dict__))
#if data process already created, check origin
if not 'data process assigned to transform worker' in data_process_event.description:
self.assertIn( data_process_event.origin, self.dp_list)
def validate_output_granule(self, msg, route, stream_id):
self.assertTrue( stream_id in [self._output_stream_one_id, self._output_stream_two_id])
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule stream_id: %s', stream_id)
if stream_id == self._output_stream_one_id:
sal_val = rdt['salinity']
log.debug('validate_output_granule sal_val: %s', sal_val)
np.testing.assert_array_equal(sal_val, np.array([3]))
self.event1_verified.set()
else:
cond_val = rdt['conductivity']
log.debug('validate_output_granule cond_val: %s', cond_val)
np.testing.assert_array_equal(cond_val, np.array([5]))
self.event2_verified.set()
def start_event_listener(self):
es = EventSubscriber(event_type=OT.DataProcessStatusEvent, callback=self.validate_event)
es.start()
self.addCleanup(es.stop)
class CtdbpTransformsIntTest(IonIntegrationTestCase):
def setUp(self):
super(CtdbpTransformsIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.dataproduct_management = DataProductManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
# This is for the time values inside the packets going into the transform
self.i = 0
# Cleanup of queue created by the subscriber
def _get_new_ctd_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
rdt['time'] = numpy.arange(self.i, self.i + length)
for field in rdt:
if isinstance(
rdt._pdict.get_context(field).param_type, QuantityType):
rdt[field] = numpy.array(
[random.uniform(0.0, 75.0) for i in xrange(length)])
g = rdt.to_granule()
self.i += length
return g
def _create_input_param_dict_for_test(self, parameter_dict_name=''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'time',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext(
'conductivity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = ''
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = ''
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temperature',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = ''
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext(
'density',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = ''
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext(
'salinity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = ''
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(
pc_k, pc[1].dump())
pc_list.append(ctxt_id)
self.addCleanup(self.dataset_management.delete_parameter_context,
ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(
parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary,
pdict_id)
return pdict_id
def test_ctdbp_L0_all(self):
"""
Test packets processed by the ctdbp_L0_all transform
"""
#----------- Data Process Definition --------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='CTDBP_L0_all',
description=
'Take parsed stream and put the C, T and P into three separate L0 streams.',
module='ion.processes.data.transforms.ctdbp.ctdbp_L0',
class_name='CTDBP_L0_all')
dprocdef_id = self.data_process_management.create_data_process_definition(
dpd_obj)
self.addCleanup(
self.data_process_management.delete_data_process_definition,
dprocdef_id)
log.debug("created data process definition: id = %s", dprocdef_id)
#----------- Data Products --------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_param_dict = self._create_input_param_dict_for_test(
parameter_dict_name='fictitious_ctdp_param_dict')
# Get the stream definition for the stream using the parameter dictionary
# input_param_dict = self.dataset_management.read_parameter_dictionary_by_name('ctdbp_cdef_sample', id_only=True)
input_stream_def_dict = self.pubsub.create_stream_definition(
name='parsed', parameter_dictionary_id=input_param_dict)
self.addCleanup(self.pubsub.delete_stream_definition,
input_stream_def_dict)
log.debug("Got the parsed parameter dictionary: id: %s",
input_param_dict)
log.debug("Got the stream def for parsed input: %s",
input_stream_def_dict)
# Input data product
parsed_stream_dp_obj = IonObject(
RT.DataProduct,
name='parsed_stream',
description='Parsed stream input to CTBP L0 transform',
temporal_domain=tdom,
spatial_domain=sdom)
input_dp_id = self.dataproduct_management.create_data_product(
data_product=parsed_stream_dp_obj,
stream_definition_id=input_stream_def_dict)
self.addCleanup(self.dataproduct_management.delete_data_product,
input_dp_id)
# output data product
L0_stream_dp_obj = IonObject(
RT.DataProduct,
name='L0_stream',
description='L0_stream output of CTBP L0 transform',
temporal_domain=tdom,
spatial_domain=sdom)
L0_stream_dp_id = self.dataproduct_management.create_data_product(
data_product=L0_stream_dp_obj,
stream_definition_id=input_stream_def_dict)
self.addCleanup(self.dataproduct_management.delete_data_product,
L0_stream_dp_id)
# We need the key name here to be "L0_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L0_stream when the config is used to launch the data process
out_stream_ids, _ = self.resource_registry.find_objects(
L0_stream_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(out_stream_ids))
output_stream_id = out_stream_ids[0]
dproc_id = self.data_process_management.create_data_process(
data_process_definition_id=dprocdef_id,
in_data_product_ids=[input_dp_id],
out_data_product_ids=[L0_stream_dp_id],
configuration=None)
self.addCleanup(self.data_process_management.delete_data_process,
dproc_id)
log.debug("Created a data process for ctdbp_L0. id: %s", dproc_id)
# Activate the data process
self.data_process_management.activate_data_process(dproc_id)
self.addCleanup(self.data_process_management.deactivate_data_process,
dproc_id)
#----------- Find the stream that is associated with the input data product when it was created by create_data_product() --------------------------------
stream_ids, _ = self.resource_registry.find_objects(
input_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(stream_ids))
input_stream_id = stream_ids[0]
stream_route = self.pubsub.read_stream_route(input_stream_id)
log.debug("The input stream for the L0 transform: %s", input_stream_id)
#----------- Create a subscriber that will listen to the transform's output --------------------------------
ar = gevent.event.AsyncResult()
def subscriber(m, r, s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub',
callback=subscriber)
sub_id = self.pubsub.create_subscription('subscriber_to_transform',
stream_ids=[output_stream_id],
exchange_name='sub')
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
#----------- Publish on that stream so that the transform can receive it --------------------------------
pub = StandaloneStreamPublisher(input_stream_id, stream_route)
publish_granule = self._get_new_ctd_packet(
stream_definition_id=input_stream_def_dict, length=5)
pub.publish(publish_granule)
log.debug("Published the following granule: %s", publish_granule)
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the transform: %s",
granule_from_transform)
# Check that the granule published by the L0 transform has the right properties
self._check_granule_from_transform(granule_from_transform)
def _check_granule_from_transform(self, granule):
"""
An internal method to check if a granule has the right properties
"""
pass
class TestDataProcessWithLookupTable(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
#@unittest.skip('not working')
def test_lookupTableProcessing(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice and attachment for lookup table
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(instDevice_id, att)
print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', deviceAttachment
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
driver_config = {
'dvr_mod' : 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls' : 'SBE37Driver',
'workdir' : '/tmp/',
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'TestDataProcessWithLookupTable: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
print 'TestDataProcessWithLookupTable: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test')
try:
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..."
att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) )
processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) )
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity')
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure')
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature')
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProcessWithLookupTable: create L0 all data_process start")
try:
in_prods = []
in_prods.append(ctd_parsed_data_product)
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, in_prods, self.output_products)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("TestDataProcessWithLookupTable: create L0 all data_process return")
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..."
att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
print 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = ', processAttachment
class CtdTransformsIntTest(IonIntegrationTestCase):
def setUp(self):
super(CtdTransformsIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.dataproduct_management = DataProductManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
# This is for the time values inside the packets going into the transform
self.i = 0
# Cleanup of queue created by the subscriber
self.queue_cleanup = []
self.data_process_cleanup = []
def _create_input_param_dict_for_test(self, parameter_dict_name=""):
pdict = ParameterDictionary()
t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=numpy.dtype("float64")))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = "seconds since 01-01-1900"
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext("conductivity", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
cond_ctxt.uom = ""
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext("pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
pres_ctxt.uom = ""
pdict.add_context(pres_ctxt)
if parameter_dict_name == "input_param_dict":
temp_ctxt = ParameterContext("temperature", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
else:
temp_ctxt = ParameterContext("temp", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
temp_ctxt.uom = ""
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext("density", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
dens_ctxt.uom = ""
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext("salinity", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
sal_ctxt.uom = ""
pdict.add_context(sal_ctxt)
# create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == "input_param_dict":
self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id)
elif parameter_dict_name == "output_param_dict" and pc[1].name == "temp":
self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
def _get_new_ctd_L0_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
rdt["time"] = numpy.arange(self.i, self.i + length)
for field in rdt:
if isinstance(rdt._pdict.get_context(field).param_type, QuantityType):
rdt[field] = numpy.array([random.uniform(0.0, 75.0) for i in xrange(length)])
g = rdt.to_granule()
self.i += length
return g
def _create_calibration_coefficients_dict(self):
config = DotDict()
config.process.calibration_coeffs = {
"temp_calibration_coeffs": {
"TA0": 1.561342e-03,
"TA1": 2.561486e-04,
"TA2": 1.896537e-07,
"TA3": 1.301189e-07,
"TOFFSET": 0.000000e00,
},
"cond_calibration_coeffs": {
"G": -9.896568e-01,
"H": 1.316599e-01,
"I": -2.213854e-04,
"J": 3.292199e-05,
"CPCOR": -9.570000e-08,
"CTCOR": 3.250000e-06,
"CSLOPE": 1.000000e00,
},
"pres_calibration_coeffs": {
"PA0": 4.960417e-02,
"PA1": 4.883682e-04,
"PA2": -5.687309e-12,
"PTCA0": 5.249802e05,
"PTCA1": 7.595719e00,
"PTCA2": -1.322776e-01,
"PTCB0": 2.503125e01,
"PTCB1": 5.000000e-05,
"PTCB2": 0.000000e00,
"PTEMPA0": -6.431504e01,
"PTEMPA1": 5.168177e01,
"PTEMPA2": -2.847757e-01,
"POFFSET": 0.000000e00,
},
}
return config
def clean_queues(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
def cleaning_operations(self):
for dproc_id in self.data_process_cleanup:
self.data_process_management.delete_data_process(dproc_id)
def test_ctd_L1_all(self):
"""
Test that packets are processed by the ctd_L1_all transform
"""
# ----------- Data Process Definition --------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="CTDBP_L1_Transform",
description="Take granules on the L0 stream which have the C, T and P data and separately apply algorithms and output on the L1 stream.",
module="ion.processes.data.transforms.ctdbp.ctdbp_L1",
class_name="CTDBP_L1_Transform",
)
dprocdef_id = self.data_process_management.create_data_process_definition(dpd_obj)
self.addCleanup(self.data_process_management.delete_data_process_definition, dprocdef_id)
log.debug("created data process definition: id = %s", dprocdef_id)
# ----------- Data Products --------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
# Get the stream definition for the stream using the parameter dictionary
L0_pdict_id = self._create_input_param_dict_for_test(parameter_dict_name="input_param_dict")
L0_stream_def_id = self.pubsub.create_stream_definition(name="parsed", parameter_dictionary_id=L0_pdict_id)
self.addCleanup(self.pubsub.delete_stream_definition, L0_stream_def_id)
L1_pdict_id = self._create_input_param_dict_for_test(parameter_dict_name="output_param_dict")
L1_stream_def_id = self.pubsub.create_stream_definition(name="L1_out", parameter_dictionary_id=L1_pdict_id)
self.addCleanup(self.pubsub.delete_stream_definition, L1_stream_def_id)
log.debug("Got the parsed parameter dictionary: id: %s", L0_pdict_id)
log.debug("Got the stream def for parsed input: %s", L0_stream_def_id)
log.debug("got the stream def for the output: %s", L1_stream_def_id)
# Input data product
L0_stream_dp_obj = IonObject(
RT.DataProduct,
name="L0_stream",
description="L0 stream input to CTBP L1 transform",
temporal_domain=tdom,
spatial_domain=sdom,
)
input_dp_id = self.dataproduct_management.create_data_product(
data_product=L0_stream_dp_obj, stream_definition_id=L0_stream_def_id
)
self.addCleanup(self.dataproduct_management.delete_data_product, input_dp_id)
# output data product
L1_stream_dp_obj = IonObject(
RT.DataProduct,
name="L1_stream",
description="L1_stream output of CTBP L1 transform",
temporal_domain=tdom,
spatial_domain=sdom,
)
L1_stream_dp_id = self.dataproduct_management.create_data_product(
data_product=L1_stream_dp_obj, stream_definition_id=L1_stream_def_id
)
self.addCleanup(self.dataproduct_management.delete_data_product, L1_stream_dp_id)
# We need the key name here to be "L1_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L1_stream when the config is used to launch the data process
out_stream_ids, _ = self.resource_registry.find_objects(L1_stream_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(out_stream_ids))
output_stream_id = out_stream_ids[0]
config = self._create_calibration_coefficients_dict()
dproc_id = self.data_process_management.create_data_process(
data_process_definition_id=dprocdef_id,
in_data_product_ids=[input_dp_id],
out_data_product_ids=[L1_stream_dp_id],
configuration=config,
)
self.addCleanup(self.data_process_management.delete_data_process, dproc_id)
log.debug("Created a data process for ctdbp_L1. id: %s", dproc_id)
# Activate the data process
self.data_process_management.activate_data_process(dproc_id)
self.addCleanup(self.data_process_management.deactivate_data_process, dproc_id)
# ----------- Find the stream that is associated with the input data product when it was created by create_data_product() --------------------------------
stream_ids, _ = self.resource_registry.find_objects(input_dp_id, PRED.hasStream, RT.Stream, True)
input_stream_id = stream_ids[0]
input_stream = self.resource_registry.read(input_stream_id)
stream_route = input_stream.stream_route
log.debug("The input stream for the L1 transform: %s", input_stream_id)
# ----------- Create a subscriber that will listen to the transform's output --------------------------------
ar = gevent.event.AsyncResult()
def subscriber(m, r, s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name="sub", callback=subscriber)
sub_id = self.pubsub.create_subscription(
"subscriber_to_transform", stream_ids=[output_stream_id], exchange_name="sub"
)
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
# ----------- Publish on that stream so that the transform can receive it --------------------------------
pub = StandaloneStreamPublisher(input_stream_id, stream_route)
publish_granule = self._get_new_ctd_L0_packet(stream_definition_id=L0_stream_def_id, length=5)
pub.publish(publish_granule)
log.debug("Published the following granule: %s", publish_granule)
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the transform: %s", granule_from_transform)
# Check that the granule published by the L1 transform has the right properties
self._check_granule_from_transform(granule_from_transform)
def _check_granule_from_transform(self, granule):
"""
An internal method to check if a granule has the right properties
"""
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assertIn("pressure", rdt)
self.assertIn("temp", rdt)
self.assertIn("conductivity", rdt)
self.assertIn("time", rdt)
class CtdTransformsIntTest(IonIntegrationTestCase):
def setUp(self):
super(CtdTransformsIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.dataproduct_management = DataProductManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
# This is for the time values inside the packets going into the transform
self.i = 0
# Cleanup of queue created by the subscriber
self.queue_cleanup = []
self.data_process_cleanup = []
def _create_input_param_dict_for_test(self, parameter_dict_name = ''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = ''
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = ''
pdict.add_context(pres_ctxt)
if parameter_dict_name == 'input_param_dict':
temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
else:
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = ''
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = ''
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = ''
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == 'input_param_dict':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
elif parameter_dict_name == 'output_param_dict' and pc[1].name == 'temp':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
def _get_new_ctd_L0_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
rdt['time'] = numpy.arange(self.i, self.i+length)
for field in rdt:
if isinstance(rdt._pdict.get_context(field).param_type, QuantityType):
rdt[field] = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)])
g = rdt.to_granule()
self.i+=length
return g
def _create_calibration_coefficients_dict(self):
config = DotDict()
config.process.calibration_coeffs = {
'temp_calibration_coeffs': {
'TA0' : 1.561342e-03,
'TA1' : 2.561486e-04,
'TA2' : 1.896537e-07,
'TA3' : 1.301189e-07,
'TOFFSET' : 0.000000e+00
},
'cond_calibration_coeffs': {
'G' : -9.896568e-01,
'H' : 1.316599e-01,
'I' : -2.213854e-04,
'J' : 3.292199e-05,
'CPCOR' : -9.570000e-08,
'CTCOR' : 3.250000e-06,
'CSLOPE' : 1.000000e+00
},
'pres_calibration_coeffs' : {
'PA0' : 4.960417e-02,
'PA1' : 4.883682e-04,
'PA2' : -5.687309e-12,
'PTCA0' : 5.249802e+05,
'PTCA1' : 7.595719e+00,
'PTCA2' : -1.322776e-01,
'PTCB0' : 2.503125e+01,
'PTCB1' : 5.000000e-05,
'PTCB2' : 0.000000e+00,
'PTEMPA0' : -6.431504e+01,
'PTEMPA1' : 5.168177e+01,
'PTEMPA2' : -2.847757e-01,
'POFFSET' : 0.000000e+00
}
}
return config
def clean_queues(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
def cleaning_operations(self):
for dproc_id in self.data_process_cleanup:
self.data_process_management.delete_data_process(dproc_id)
def test_ctd_L1_all(self):
"""
Test that packets are processed by the ctd_L1_all transform
"""
#----------- Data Process Definition --------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='CTDBP_L1_Transform',
description='Take granules on the L0 stream which have the C, T and P data and separately apply algorithms and output on the L1 stream.',
module='ion.processes.data.transforms.ctdbp.ctdbp_L1',
class_name='CTDBP_L1_Transform')
dprocdef_id = self.data_process_management.create_data_process_definition(dpd_obj)
self.addCleanup(self.data_process_management.delete_data_process_definition, dprocdef_id)
log.debug("created data process definition: id = %s", dprocdef_id)
#----------- Data Products --------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
# Get the stream definition for the stream using the parameter dictionary
L0_pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'input_param_dict')
L0_stream_def_id = self.pubsub.create_stream_definition(name='parsed', parameter_dictionary_id=L0_pdict_id)
self.addCleanup(self.pubsub.delete_stream_definition, L0_stream_def_id)
L1_pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'output_param_dict')
L1_stream_def_id = self.pubsub.create_stream_definition(name='L1_out', parameter_dictionary_id=L1_pdict_id)
self.addCleanup(self.pubsub.delete_stream_definition, L1_stream_def_id)
log.debug("Got the parsed parameter dictionary: id: %s", L0_pdict_id)
log.debug("Got the stream def for parsed input: %s", L0_stream_def_id)
log.debug("got the stream def for the output: %s", L1_stream_def_id)
# Input data product
L0_stream_dp_obj = IonObject(RT.DataProduct,
name='L0_stream',
description='L0 stream input to CTBP L1 transform',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproduct_management.create_data_product(data_product=L0_stream_dp_obj,
stream_definition_id=L0_stream_def_id
)
self.addCleanup(self.dataproduct_management.delete_data_product, input_dp_id)
# output data product
L1_stream_dp_obj = IonObject(RT.DataProduct,
name='L1_stream',
description='L1_stream output of CTBP L1 transform',
temporal_domain = tdom,
spatial_domain = sdom)
L1_stream_dp_id = self.dataproduct_management.create_data_product(data_product=L1_stream_dp_obj,
stream_definition_id=L1_stream_def_id
)
self.addCleanup(self.dataproduct_management.delete_data_product, L1_stream_dp_id)
# We need the key name here to be "L1_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L1_stream when the config is used to launch the data process
self.output_products = {'L1_stream' : L1_stream_dp_id}
out_stream_ids, _ = self.resource_registry.find_objects(L1_stream_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(out_stream_ids))
output_stream_id = out_stream_ids[0]
config = self._create_calibration_coefficients_dict()
dproc_id = self.data_process_management.create_data_process( dprocdef_id, [input_dp_id], self.output_products, config)
self.addCleanup(self.data_process_management.delete_data_process, dproc_id)
log.debug("Created a data process for ctdbp_L1. id: %s", dproc_id)
# Activate the data process
self.data_process_management.activate_data_process(dproc_id)
self.addCleanup(self.data_process_management.deactivate_data_process, dproc_id)
#----------- Find the stream that is associated with the input data product when it was created by create_data_product() --------------------------------
stream_ids, _ = self.resource_registry.find_objects(input_dp_id, PRED.hasStream, RT.Stream, True)
input_stream_id = stream_ids[0]
input_stream = self.resource_registry.read(input_stream_id)
stream_route = input_stream.stream_route
log.debug("The input stream for the L1 transform: %s", input_stream_id)
#----------- Create a subscriber that will listen to the transform's output --------------------------------
ar = gevent.event.AsyncResult()
def subscriber(m,r,s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub', callback=subscriber)
sub_id = self.pubsub.create_subscription('subscriber_to_transform',
stream_ids=[output_stream_id],
exchange_name='sub')
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
#----------- Publish on that stream so that the transform can receive it --------------------------------
pub = StandaloneStreamPublisher(input_stream_id, stream_route)
publish_granule = self._get_new_ctd_L0_packet(stream_definition_id=L0_stream_def_id, length = 5)
pub.publish(publish_granule)
log.debug("Published the following granule: %s", publish_granule)
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the transform: %s", granule_from_transform)
# Check that the granule published by the L1 transform has the right properties
self._check_granule_from_transform(granule_from_transform)
def _check_granule_from_transform(self, granule):
"""
An internal method to check if a granule has the right properties
"""
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assertIn('pressure', rdt)
self.assertIn('temp', rdt)
self.assertIn('conductivity', rdt)
self.assertIn('time', rdt)
class TestTransformPrime(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Because hey why not?!
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
def _create_proc_def(self):
dpd_obj = DataProcessDefinition(
name='Optimus',
description='It\'s a transformer',
module='ion.processes.data.transforms.transform_prime',
class_name='TransformPrime')
return self.data_process_management.create_data_process_definition(dpd_obj)
def _L0_pdict(self):
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump(), parameter_type='quantity<int64>', unit_of_measure=t_ctxt.uom)
lat_ctxt = ParameterContext('lat', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = -9999
lat_ctxt_id = self.dataset_management.create_parameter_context(name='lat', parameter_context=lat_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lat_ctxt.uom)
lon_ctxt = ParameterContext('lon', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = -9999
lon_ctxt_id = self.dataset_management.create_parameter_context(name='lon', parameter_context=lon_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lon_ctxt.uom)
temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')))
temp_ctxt.uom = 'deg_C'
temp_ctxt.fill_value = -9999
temp_ctxt_id = self.dataset_management.create_parameter_context(name='TEMPWAT_L0', parameter_context=temp_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=temp_ctxt.uom)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')))
cond_ctxt.uom = 'S m-1'
cond_ctxt.fill_value = -9999
cond_ctxt_id = self.dataset_management.create_parameter_context(name='CONDWAT_L0', parameter_context=cond_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=cond_ctxt.uom)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')))
press_ctxt.uom = 'dbar'
press_ctxt.fill_value = -9999
press_ctxt_id = self.dataset_management.create_parameter_context(name='PRESWAT_L0', parameter_context=press_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=press_ctxt.uom)
context_ids = [t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id]
pdict_id = self.dataset_management.create_parameter_dictionary('L0 SBE37', parameter_context_ids=context_ids, temporal_context='time')
return pdict_id
def _L1_pdict(self):
pdict_id = self._L0_pdict()
param_context_ids = self.dataset_management.read_parameter_contexts(pdict_id,id_only=True)
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(TEMPWAT_L0 / 10000) - 10'
tl1_pmap = {'TEMPWAT_L0':'TEMPWAT_L0'}
func = NumexprFunction('TEMPWAT_L1', tl1_func, tl1_pmap)
tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(name=tempL1_ctxt.name, parameter_context=tempL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=tempL1_ctxt.uom)
param_context_ids.append(tempL1_ctxt_id)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(CONDWAT_L0 / 100000) - 0.5'
cl1_pmap = {'CONDWAT_L0':'CONDWAT_L0'}
func = NumexprFunction('CONDWAT_L1', cl1_func, cl1_pmap)
condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(name=condL1_ctxt.name, parameter_context=condL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=condL1_ctxt.uom)
param_context_ids.append(condL1_ctxt_id)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(PRESWAT_L0 * 679.34040721 / (0.85 * 65536)) - (0.05 * 679.34040721)'
pl1_pmap = {'PRESWAT_L0':'PRESWAT_L0'}
func = NumexprFunction('PRESWAT_L1', pl1_func, pl1_pmap)
presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(name=presL1_ctxt.name, parameter_context=presL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=presL1_ctxt.uom)
param_context_ids.append(presL1_ctxt_id)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = [NumexprFunction('CONDWAT_L1*10', 'C*10', {'C':'CONDWAT_L1'}), 'TEMPWAT_L1', 'PRESWAT_L1']
sal_kwargmap = None
func = PythonFunction('PRACSAL', owner, sal_func, sal_arglist, sal_kwargmap)
sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(func), variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type='pfunc', unit_of_measure=sal_ctxt.uom)
param_context_ids.append(sal_ctxt_id)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_func = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'lon','lat'], None)
#abs_sal_func = PythonFunction('abs_sal', owner, 'SA_from_SP', ['lon','lat'], None)
cons_temp_func = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_func, 'TEMPWAT_L1', 'PRESWAT_L1'], None)
dens_func = PythonFunction('DENSITY', owner, 'rho', [abs_sal_func, cons_temp_func, 'PRESWAT_L1'], None)
dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_func), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name=dens_ctxt.name, parameter_context=dens_ctxt.dump(), parameter_type='pfunc', unit_of_measure=dens_ctxt.uom)
param_context_ids.append(dens_ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary('L1_SBE37', parameter_context_ids=param_context_ids, temporal_context='time')
return pdict_id
def _data_product(self, name, stream_def, exchange_pt):
tdom, sdom = time_series_domain()
dp_obj = DataProduct(name=name, description='blah', spatial_domain=sdom.dump(), temporal_domain=tdom.dump())
dp_id = self.data_product_management.create_data_product(dp_obj, stream_def, exchange_pt)
return dp_id
def _data_process(self, proc_def_id, input_products, output_product, stream_def):
fake_producer = DataProducer(name='fake_producer')
fake_producer_id, _ = self.container.resource_registry.create(fake_producer)
self.data_process_management.assign_stream_definition_to_data_process_definition(stream_def,proc_def_id,binding='output')
data_process_id = self.data_process_management.create_data_process(proc_def_id, input_products, {'output':output_product})
self.container.resource_registry.create_association(subject=data_process_id, predicate=PRED.hasDataProducer, object=fake_producer_id)
self.data_process_management.activate_data_process(data_process_id)
def _fake_producer(self):
if not hasattr(self, 'producer'):
self.fake_producer_id,_ = self.container.resource_registry.create(DataProducer(name='fake_producer'))
return self.fake_producer_id
def _publisher(self, data_product_id):
stream_ids, _ = self.container.resource_registry.find_resources(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
route = self.pubsub_management.read_stream_route(stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
return publisher
def _get_param_vals(self, name, slice_, dims):
shp = utils.slice_shape(slice_, dims)
def _getarr(vmin, shp, vmax=None,):
if vmax is None:
return np.empty(shp).fill(vmin)
return np.arange(vmin, vmax, (vmax - vmin) / int(utils.prod(shp)), dtype='float32').reshape(shp)
if name == 'LAT':
ret = np.empty(shp)
ret.fill(45)
elif name == 'LON':
ret = np.empty(shp)
ret.fill(-71)
elif name == 'TEMPWAT_L0':
ret = _getarr(280000, shp, 350000)
elif name == 'CONDWAT_L0':
ret = _getarr(100000, shp, 750000)
elif name == 'PRESWAT_L0':
ret = _getarr(3000, shp, 10000)
elif name in self.value_classes: # Non-L0 parameters
ret = self.value_classes[name][:]
else:
return np.zeros(shp)
return ret
def _setup_streams(self, exchange_pt1, exchange_pt2, available_fields_in=[], available_fields_out=[]):
proc_def_id = self._create_proc_def()
incoming_pdict_id = self._L0_pdict()
outgoing_pdict_id = self._L1_pdict()
incoming_stream_def_id = self.pubsub_management.create_stream_definition('L0_stream_def', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in)
outgoing_stream_def_id = self.pubsub_management.create_stream_definition('L1_stream_def', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out)
L0_data_product_id = self._data_product('L0_SBE37', incoming_stream_def_id, exchange_pt1)
L1_data_product_id = self._data_product('L1_SBE37', outgoing_stream_def_id, exchange_pt2)
self._data_process(proc_def_id, [L0_data_product_id], L1_data_product_id, outgoing_stream_def_id)
stream_ids, _ = self.container.resource_registry.find_objects(L0_data_product_id, PRED.hasStream, None, True)
stream_id_in = stream_ids[0]
stream_ids, _ = self.container.resource_registry.find_objects(L1_data_product_id, PRED.hasStream, None, True)
stream_id_out = stream_ids[0]
stream_route_in = self.pubsub_management.read_stream_route(stream_id_in)
stream_route_out = self.pubsub_management.read_stream_route(stream_id_out)
return (stream_id_in,stream_id_out,stream_route_in,stream_route_out,incoming_stream_def_id,outgoing_stream_def_id)
def _validate_transforms(self, rdt_in, rdt_out):
#passthrus
self.assertTrue(np.allclose(rdt_in['time'], rdt_out['time']))
self.assertTrue(np.allclose(rdt_in['lat'], rdt_out['lat']))
self.assertTrue(np.allclose(rdt_in['lon'], rdt_out['lon']))
self.assertTrue(np.allclose(rdt_in['TEMPWAT_L0'], rdt_out['TEMPWAT_L0']))
self.assertTrue(np.allclose(rdt_in['CONDWAT_L0'], rdt_out['CONDWAT_L0']))
self.assertTrue(np.allclose(rdt_in['PRESWAT_L0'], rdt_out['PRESWAT_L0']))
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
t1 = (rdt_out['TEMPWAT_L0'] / 10000) - 10
self.assertTrue(np.allclose(rdt_out['TEMPWAT_L1'], t1))
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
c1 = (rdt_out['CONDWAT_L0'] / 100000) - 0.5
self.assertTrue(np.allclose(rdt_out['CONDWAT_L1'], c1))
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
p1 = (rdt_out['PRESWAT_L0'] * 679.34040721 / (0.85 * 65536)) - (0.05 * 679.34040721)
self.assertTrue(np.allclose(rdt_out['PRESWAT_L1'], p1))
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
ps = gsw.SP_from_C((rdt_out['CONDWAT_L1'] * 10.), rdt_out['TEMPWAT_L1'], rdt_out['PRESWAT_L1'])
self.assertTrue(np.allclose(rdt_out['PRACSAL'], ps))
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
abs_sal = gsw.SA_from_SP(rdt_out['PRACSAL'], rdt_out['PRESWAT_L1'], rdt_out['lon'], rdt_out['lat'])
cons_temp = gsw.CT_from_t(abs_sal, rdt_out['TEMPWAT_L1'], rdt_out['PRESWAT_L1'])
rho = gsw.rho(abs_sal, cons_temp, rdt_out['PRESWAT_L1'])
self.assertTrue(np.allclose(rdt_out['DENSITY'], rho))
def test_execute_transform(self):
available_fields_in = ['time', 'lat', 'lon', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']
available_fields_out = ['time', 'lat', 'lon', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0', 'TEMPWAT_L1','CONDWAT_L1','PRESWAT_L1','PRACSAL', 'DENSITY']
exchange_pt1 = 'xp1'
exchange_pt2 = 'xp2'
stream_id_in,stream_id_out,stream_route_in,stream_route_out,stream_def_in_id,stream_def_out_id = self._setup_streams(exchange_pt1, exchange_pt2, available_fields_in, available_fields_out)
rdt_in = RecordDictionaryTool(stream_definition_id=stream_def_in_id)
dt = 20
rdt_in['time'] = np.arange(dt)
rdt_in['lat'] = [40.992469] * dt
rdt_in['lon'] = [-71.727069] * dt
rdt_in['TEMPWAT_L0'] = self._get_param_vals('TEMPWAT_L0', slice(None), (dt,))
rdt_in['CONDWAT_L0'] = self._get_param_vals('CONDWAT_L0', slice(None), (dt,))
rdt_in['PRESWAT_L0'] = self._get_param_vals('PRESWAT_L0', slice(None), (dt,))
msg = rdt_in.to_granule()
#pid = self.container.spawn_process('transform_stream','ion.processes.data.transforms.transform_prime','TransformPrime',{'process':{'routes':{(stream_id_in, stream_id_out):None},'stream_id':stream_id_out}})
config = {'process':{'routes':{(stream_id_in, stream_id_out):None},'queue_name':exchange_pt1, 'publish_streams':{str(stream_id_out):stream_id_out}, 'process_type':'stream_process'}}
pid = self.container.spawn_process('transform_stream','ion.processes.data.transforms.transform_prime','TransformPrime',config)
rdt_out = self.container.proc_manager.procs[pid]._execute_transform(msg, (stream_id_in,stream_id_out))
#need below to wrap result in a param val object
rdt_out = RecordDictionaryTool.load_from_granule(rdt_out.to_granule())
for k,v in rdt_out.iteritems():
self.assertEqual(len(v), dt)
self._validate_transforms(rdt_in, rdt_out)
self.container.proc_manager.terminate_process(pid)
def test_transform_prime_no_available_fields(self):
available_fields_in = []
available_fields_out = []
exchange_pt1 = 'xp1'
exchange_pt2 = 'xp2'
stream_id_in,stream_id_out,stream_route_in,stream_route_out,stream_def_in_id,stream_def_out_id = self._setup_streams(exchange_pt1, exchange_pt2, available_fields_in, available_fields_out)
#launch transform
config = {'process':{'routes':{(stream_id_in, stream_id_out):None},'queue_name':exchange_pt1, 'publish_streams':{str(stream_id_out):stream_id_out}, 'process_type':'stream_process'}}
pid = self.container.spawn_process('transform_stream','ion.processes.data.transforms.transform_prime','TransformPrime',config)
#create publish
publisher = StandaloneStreamPublisher(stream_id_in, stream_route_in)
self.container.proc_manager.procs[pid].subscriber.xn.bind(stream_route_in.routing_key, publisher.xp)
#data
rdt_in = RecordDictionaryTool(stream_definition_id=stream_def_in_id)
dt = 20
rdt_in['time'] = np.arange(dt)
rdt_in['lat'] = [40.992469] * dt
rdt_in['lon'] = [-71.727069] * dt
rdt_in['TEMPWAT_L0'] = self._get_param_vals('TEMPWAT_L0', slice(None), (dt,))
rdt_in['CONDWAT_L0'] = self._get_param_vals('CONDWAT_L0', slice(None), (dt,))
rdt_in['PRESWAT_L0'] = self._get_param_vals('PRESWAT_L0', slice(None), (dt,))
msg = rdt_in.to_granule()
#publish granule to transform and have transform publish it to subsciber
#validate transformed data
e = gevent.event.Event()
def cb(msg, sr, sid):
self.assertEqual(sid, stream_id_out)
rdt_out = RecordDictionaryTool.load_from_granule(msg)
self.assertEquals(set([k for k,v in rdt_out.iteritems()]), set(available_fields_out))
for k,v in rdt_out.iteritems():
self.assertEquals(rdt_out[k], None)
e.set()
sub = StandaloneStreamSubscriber('stream_subscriber', cb)
sub.xn.bind(stream_route_out.routing_key, getattr(self.container.proc_manager.procs[pid], stream_id_out).xp)
self.addCleanup(sub.stop)
sub.start()
#publish msg to transform
publisher.publish(msg)
#wait to receive msg
self.assertTrue(e.wait(4))
#self.container.proc_manager.terminate_process(pid)
def test_transform_prime(self):
available_fields_in = ['time', 'lat', 'lon', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']
available_fields_out = ['time', 'lat', 'lon', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0', 'TEMPWAT_L1','CONDWAT_L1','PRESWAT_L1','PRACSAL', 'DENSITY']
exchange_pt1 = 'xp1'
exchange_pt2 = 'xp2'
stream_id_in,stream_id_out,stream_route_in,stream_route_out,stream_def_in_id,stream_def_out_id = self._setup_streams(exchange_pt1, exchange_pt2, available_fields_in, available_fields_out)
#launch transform
config = {'process':{'routes':{(stream_id_in, stream_id_out):None},'queue_name':exchange_pt1, 'publish_streams':{str(stream_id_out):stream_id_out}, 'process_type':'stream_process'}}
pid = self.container.spawn_process('transform_stream','ion.processes.data.transforms.transform_prime','TransformPrime',config)
#create publish
publisher = StandaloneStreamPublisher(stream_id_in, stream_route_in)
self.container.proc_manager.procs[pid].subscriber.xn.bind(stream_route_in.routing_key, publisher.xp)
#data
rdt_in = RecordDictionaryTool(stream_definition_id=stream_def_in_id)
dt = 20
rdt_in['time'] = np.arange(dt)
rdt_in['lat'] = [40.992469] * dt
rdt_in['lon'] = [-71.727069] * dt
rdt_in['TEMPWAT_L0'] = self._get_param_vals('TEMPWAT_L0', slice(None), (dt,))
rdt_in['CONDWAT_L0'] = self._get_param_vals('CONDWAT_L0', slice(None), (dt,))
rdt_in['PRESWAT_L0'] = self._get_param_vals('PRESWAT_L0', slice(None), (dt,))
msg = rdt_in.to_granule()
#publish granule to transform and have transform publish it to subsciber
#validate transformed data
e = gevent.event.Event()
def cb(msg, sr, sid):
self.assertEqual(sid, stream_id_out)
rdt_out = RecordDictionaryTool.load_from_granule(msg)
self.assertEquals(set([k for k,v in rdt_out.iteritems()]), set(available_fields_out))
self._validate_transforms(rdt_in, rdt_out)
e.set()
sub = StandaloneStreamSubscriber('stream_subscriber', cb)
sub.xn.bind(stream_route_out.routing_key, getattr(self.container.proc_manager.procs[pid], stream_id_out).xp)
self.addCleanup(sub.stop)
sub.start()
#publish msg to transform
publisher.publish(msg)
#wait to receive msg
self.assertTrue(e.wait(4))
class TestWorkflowManagementIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def _create_ctd_input_stream_and_data_product(self, data_product_name='ctd_parsed'):
cc = self.container
assertions = self.assertTrue
#-------------------------------
# Create CTD Parsed as the initial data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=self.ctd_stream_def, name='Simulated CTD data')
log.debug('Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,name=data_product_name,description='ctd stream test')
try:
ctd_parsed_data_product_id = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except Exception as ex:
self.fail("failed to create new data product: %s" %ex)
log.debug('new ctd_parsed_data_product_id = ', ctd_parsed_data_product_id)
#Only ever need one device for testing purposes.
instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='SBE37IMDevice')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product_id, persist_data=False, persist_metadata=False)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
ctd_stream_id = stream_ids[0]
return ctd_stream_id, ctd_parsed_data_product_id
def _start_simple_input_stream_process(self, ctd_stream_id):
return self._start_input_stream_process(ctd_stream_id)
def _start_sinusoidal_input_stream_process(self, ctd_stream_id):
return self._start_input_stream_process(ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher', 'SinusoidalCtdPublisher')
def _start_input_stream_process(self, ctd_stream_id, module = 'ion.processes.data.ctd_stream_publisher', class_name= 'SimpleCtdPublisher'):
###
### Start the process for producing the CTD data
###
# process definition for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':module,
'class':class_name
}
ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(process_definition=producer_definition)
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':ctd_stream_id,
}
}
ctd_sim_pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration)
return ctd_sim_pid
def _start_output_stream_listener(self, data_product_stream_ids, message_count_per_stream=10):
cc = self.container
assertions = self.assertTrue
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name = 'workflow_test',
name = "test workflow transformations",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn(' data received!')
results.append(message)
if len(results) >= len(data_product_stream_ids) * message_count_per_stream: #Only wait for so many messages - per stream
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='workflow_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=30))
self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def _validate_messages(self, results):
cc = self.container
assertions = self.assertTrue
first_salinity_values = None
for message in results:
try:
psd = PointSupplementStreamParser(stream_definition=self.ctd_stream_def, stream_granule=message)
temp = psd.get_values('temperature')
log.info(psd.list_field_names())
except KeyError as ke:
temp = None
if temp is not None:
assertions(isinstance(temp, numpy.ndarray))
log.info( 'temperature=' + str(numpy.nanmin(temp)))
first_salinity_values = None
else:
psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message)
log.info( psd.list_field_names())
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
log.info( 'salinity=' + str(numpy.nanmin(salinity)))
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
if first_salinity_values is None:
first_salinity_values = salinity.tolist()
else:
second_salinity_values = salinity.tolist()
assertions(len(first_salinity_values) == len(second_salinity_values))
for idx in range(0,len(first_salinity_values)):
assertions(first_salinity_values[idx]*2.0 == second_salinity_values[idx])
def _create_salinity_data_process_definition(self):
# Salinity: Data Process Definition
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='ctd_salinity')
if len(dpd) > 0:
return dpd[0]
log.debug("Create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_salinity',
description='create a salinity data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Excpetion as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='Salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(sal_stream_def_id, ctd_L2_salinity_dprocdef_id )
return ctd_L2_salinity_dprocdef_id
def _create_salinity_doubler_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler')
if len(dpd) > 0:
return dpd[0]
# Salinity Doubler: Data Process Definition
log.debug("Create data process definition SalinityDoublerTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='salinity_doubler',
description='create a salinity doubler data product',
module='ion.processes.data.transforms.example_double_salinity',
class_name='SalinityDoubler',
process_source='SalinityDoubler source code here...')
try:
salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new SalinityDoubler data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(salinity_double_stream_def_id, salinity_doubler_dprocdef_id )
return salinity_doubler_dprocdef_id
def create_transform_process(self, data_process_definition_id, data_process_input_dp_id):
data_process_definition = self.rrclient.read(data_process_definition_id)
# Find the link between the output Stream Definition resource and the Data Process Definition resource
stream_ids,_ = self.rrclient.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True)
if not stream_ids:
raise Inconsistent("The data process definition %s is missing an association to an output stream definition" % data_process_definition._id )
process_output_stream_def_id = stream_ids[0]
#Concatenate the name of the workflow and data process definition for the name of the data product output
data_process_name = data_process_definition.name
# Create the output data product of the transform
transform_dp_obj = IonObject(RT.DataProduct, name=data_process_name,description=data_process_definition.description)
transform_dp_id = self.dataproductclient.create_data_product(transform_dp_obj, process_output_stream_def_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=transform_dp_id, persist_data=True, persist_metadata=True)
#last one out of the for loop is the output product id
output_data_product_id = transform_dp_id
# Create the transform data process
log.debug("create data_process and start it")
data_process_id = self.dataprocessclient.create_data_process(data_process_definition._id, [data_process_input_dp_id], {'output':transform_dp_id})
self.dataprocessclient.activate_data_process(data_process_id)
#Find the id of the output data stream
stream_ids, _ = self.rrclient.find_objects(transform_dp_id, PRED.hasStream, None, True)
if not stream_ids:
raise Inconsistent("The data process %s is missing an association to an output stream" % data_process_id )
return data_process_id, output_data_product_id
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_SA_transform_components(self):
assertions = self.assertTrue
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product()
data_product_stream_ids.append(ctd_stream_id)
###
### Setup the first transformation
###
# Salinity: Data Process Definition
ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition()
l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process(ctd_L2_salinity_dprocdef_id,ctd_parsed_data_product_id )
## get the stream id for the transform outputs
stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
sal_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_stream_id)
###
### Setup the second transformation
###
# Salinity Doubler: Data Process Definition
salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition()
salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process(salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id )
stream_ids, _ = self.rrclient.find_objects(salinity_doubler_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
sal_dbl_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_dbl_stream_id)
#Start the input stream process
ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id)
#Start te output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the transform processes
self.dataprocessclient.deactivate_data_process(salinity_double_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_messages(results)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Salinity_Test_Workflow',description='tests a workflow of multiple transform data processes')
workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name
#Add a transformation process definition
ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False) #Don't persist the intermediate data product
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Add a transformation process definition
salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=salinity_doubler_dprocdef_id, output_data_product_name=workflow_data_product_name)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)
aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition)
assertions(len(aids) == 2 )
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product()
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, timeout=30)
workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1 )
#Verify the output data product name matches what was specified in the workflow definition
workflow_product = self.rrclient.read(workflow_product_id)
assertions(workflow_product.name == workflow_data_product_name)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 2 )
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) == 1 )
data_product_stream_ids.append(stream_ids[0])
#Start the input stream process
ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id)
#Start the output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id, False, timeout=15) # Should test true at some point
#Make sure the Workflow object was removed
objs, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(objs) == 0)
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_messages(results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
def _create_google_dt_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform')
if len(dpd) > 0:
return dpd[0]
# Data Process Definition
log.debug("Create data process definition GoogleDtTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='google_dt_transform',
description='Convert data streams to Google DataTables',
module='ion.processes.data.transforms.viz.google_dt',
class_name='VizTransformGoogleDT',
process_source='VizTransformGoogleDT source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new VizTransformGoogleDT data process definition: %s" %ex)
# create a stream definition for the data from the
stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformGoogleDT.outgoing_stream_def, name='VizTransformGoogleDT')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id )
return procdef_id
def _validate_google_dt_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
gdt_component = rdt['google_dt_components'][0]
assertions(gdt_component['viz_product_type'] == 'google_realtime_dt' )
gdt_description = gdt_component['data_table_description']
gdt_content = gdt_component['data_table_content']
assertions(gdt_description[0][0] == 'time')
assertions(len(gdt_description) > 1)
assertions(len(gdt_content) >= 0)
return
def _create_mpl_graphs_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform')
if len(dpd) > 0:
return dpd[0]
#Data Process Definition
log.debug("Create data process definition MatplotlibGraphsTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='mpl_graphs_transform',
description='Convert data streams to Matplotlib graphs',
module='ion.processes.data.transforms.viz.matplotlib_graphs',
class_name='VizTransformMatplotlibGraphs',
process_source='VizTransformMatplotlibGraphs source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new VizTransformMatplotlibGraphs data process definition: %s" %ex)
# create a stream definition for the data
stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformMatplotlibGraphs.outgoing_stream_def, name='VizTransformMatplotlibGraphs')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id )
return procdef_id
def _validate_mpl_graphs_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
graphs = rdt['matplotlib_graphs']
for graph in graphs:
assertions(graph['viz_product_type'] == 'matplotlib_graphs' )
# check to see if the list (numpy array) contians actual images
assertions(imghdr.what(graph['image_name'], graph['image_obj']) == 'png')
return
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
#unittest.skip("Skipping for debugging ")
def test_google_dt_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='GoogleDT_Test_Workflow',description='Tests the workflow of converting stream data to Google DT')
#Add a transformation process definition
google_dt_procdef_id = self._create_google_dt_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product()
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id)
workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1 )
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1 )
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) == 1 )
data_product_stream_ids.append(stream_ids[0])
#Start the input stream process
ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id)
#Start the output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id, False) # Should test true at some point
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_google_dt_results(VizTransformGoogleDT.outgoing_stream_def, results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_mpl_graphs_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Mpl_Graphs_Test_Workflow',description='Tests the workflow of converting stream data to Matplotlib graphs')
#Add a transformation process definition
mpl_graphs_procdef_id = self._create_mpl_graphs_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=mpl_graphs_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product()
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id)
workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1 )
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1 )
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) == 1 )
data_product_stream_ids.append(stream_ids[0])
#Start the input stream process
ctd_sim_pid = self._start_sinusoidal_input_stream_process(ctd_stream_id)
#Start the output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id, False) # Should test true at some point
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_mpl_graphs_results(VizTransformGoogleDT.outgoing_stream_def, results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_multiple_workflow_instances(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Multiple_Test_Workflow',description='Tests the workflow of converting stream data')
#Add a transformation process definition
google_dt_procdef_id = self._create_google_dt_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the first input data product
ctd_stream_id1, ctd_parsed_data_product_id1 = self._create_ctd_input_stream_and_data_product('ctd_parsed1')
data_product_stream_ids.append(ctd_stream_id1)
#Create and start the first workflow
workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id1)
#Create the second input data product
ctd_stream_id2, ctd_parsed_data_product_id2 = self._create_ctd_input_stream_and_data_product('ctd_parsed2')
data_product_stream_ids.append(ctd_stream_id2)
#Create and start the first workflow
workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id2)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_ids,_ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 2 )
#Start the first input stream process
ctd_sim_pid1 = self._start_sinusoidal_input_stream_process(ctd_stream_id1)
#Start the second input stream process
ctd_sim_pid2 = self._start_simple_input_stream_process(ctd_stream_id2)
#Start the output stream listener to monitor a set number of messages being sent through the workflows
results = self._start_output_stream_listener(data_product_stream_ids, message_count_per_stream=5)
# stop the flow of messages...
self.process_dispatcher.cancel_process(ctd_sim_pid1) # kill the ctd simulator process - that is enough data
self.process_dispatcher.cancel_process(ctd_sim_pid2)
#Stop the first workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id1, False) # Should test true at some point
#Stop the second workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id2, False) # Should test true at some point
workflow_ids,_ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 0 )
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
aid_list = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition)
assertions(len(aid_list) == 0 )
class TestTransformWorkerSubscriptions(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_multi_subscriptions(self):
self.dp_list = []
self.event1_verified = Event()
self.event2_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product_one',
description='input test stream one')
self.input_dp_one_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product_two',
description='input test stream two')
self.input_dp_two_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
#retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_one_id, PRED.hasStream, RT.Stream, True)
self.stream_one_id = stream_ids[0]
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_two_id, PRED.hasStream, RT.Stream, True)
self.stream_two_id = stream_ids[0]
dpd_id = self.create_data_process_definition()
dp1_func_output_dp_id, dp2_func_output_dp_id = self.create_output_data_products(
)
first_dp_id = self.create_data_process_one(dpd_id,
dp1_func_output_dp_id)
second_dp_id = self.create_data_process_two(dpd_id,
self.input_dp_two_id,
dp2_func_output_dp_id)
#retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=first_dp_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
#create subscription to stream ONE, create data process and publish granule on stream ONE
#create a queue to catch the published granules of stream ONE
self.subscription_one_id = self.pubsub_client.create_subscription(
name='parsed_subscription_one',
stream_ids=[self.stream_one_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_one_id)
self.pubsub_client.activate_subscription(self.subscription_one_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_one_id)
stream_route_one = self.pubsub_client.read_stream_route(
self.stream_one_id)
self.publisher_one = StandaloneStreamPublisher(
stream_id=self.stream_one_id, stream_route=stream_route_one)
self.start_event_listener()
#data process 1 adds conductivity + pressure and puts the result in salinity
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher_one.publish(msg=rdt.to_granule(),
stream_id=self.stream_one_id)
#retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=second_dp_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
#create subscription to stream ONE and TWO, move TW subscription, create data process and publish granule on stream TWO
#create a queue to catch the published granules of stream TWO
self.subscription_two_id = self.pubsub_client.create_subscription(
name='parsed_subscription_one_two',
stream_ids=[self.stream_two_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_two_id)
self.pubsub_client.activate_subscription(self.subscription_two_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_two_id)
stream_route_two = self.pubsub_client.read_stream_route(
self.stream_two_id)
self.publisher_two = StandaloneStreamPublisher(
stream_id=self.stream_two_id, stream_route=stream_route_two)
#data process 1 adds conductivity + pressure and puts the result in salinity
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher_one.publish(msg=rdt.to_granule(),
stream_id=self.stream_one_id)
#data process 2 adds salinity + pressure and puts the result in conductivity
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [22]
rdt['pressure'] = [4]
rdt['salinity'] = [1]
self.publisher_two.publish(msg=rdt.to_granule(),
stream_id=self.stream_two_id)
self.assertTrue(self.event2_verified.wait(self.wait_time))
self.assertTrue(self.event1_verified.wait(self.wait_time))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_two_transforms_inline(self):
self.dp_list = []
self.event1_verified = Event()
self.event2_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product_one',
description='input test stream one')
self.input_dp_one_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
dpd_id = self.create_data_process_definition()
dp1_func_output_dp_id, dp2_func_output_dp_id = self.create_output_data_products(
)
first_dp_id = self.create_data_process_one(dpd_id,
dp1_func_output_dp_id)
second_dp_id = self.create_data_process_two(dpd_id,
dp1_func_output_dp_id,
dp2_func_output_dp_id)
#retrieve subscription from data process one
subscription_objs, _ = self.rrclient.find_objects(
subject=first_dp_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
#retrieve the Stream for these data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_one_id, PRED.hasStream, RT.Stream, True)
self.stream_one_id = stream_ids[0]
#the input to data process two is the output from data process one
stream_ids, assoc_ids = self.rrclient.find_objects(
dp1_func_output_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_two_id = stream_ids[0]
# Run provenance on the output dataproduct of the second data process to see all the links
# are as expected
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
dp2_func_output_dp_id)
# Do a basic check to see if there were 2 entries in the provenance graph. Parent and Child.
self.assertTrue(len(output_data_product_provenance) == 3)
# confirm that the linking from the output dataproduct to input dataproduct is correct
self.assertTrue(
dp1_func_output_dp_id in
output_data_product_provenance[dp2_func_output_dp_id]['parents'])
self.assertTrue(
self.input_dp_one_id in
output_data_product_provenance[dp1_func_output_dp_id]['parents'])
#create subscription to stream ONE, create data process and publish granule on stream ONE
#create a queue to catch the published granules of stream ONE
subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_one_id, self.stream_two_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
subscription_id)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
subscription_id)
stream_route_one = self.pubsub_client.read_stream_route(
self.stream_one_id)
self.publisher_one = StandaloneStreamPublisher(
stream_id=self.stream_one_id, stream_route=stream_route_one)
#retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=second_dp_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
#data process 1 adds conductivity + pressure and puts the result in salinity
#data process 2 adds salinity + pressure and puts the result in conductivity
self.start_event_listener()
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher_one.publish(msg=rdt.to_granule(),
stream_id=self.stream_one_id)
self.assertTrue(self.event2_verified.wait(self.wait_time))
self.assertTrue(self.event1_verified.wait(self.wait_time))
def create_data_process_definition(self):
#two data processes using one transform and one DPD
# Set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
return add_array_dpd_id
def create_data_process_one(self, data_process_definition_id,
output_dataproduct):
# Create the data process
#data process 1 adds conductivity + pressure and puts the result in salinity
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "salinity"
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=data_process_definition_id,
inputs=[self.input_dp_one_id],
outputs=[output_dataproduct],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp1_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
dp1_data_process_id)
self.dp_list.append(dp1_data_process_id)
return dp1_data_process_id
def create_data_process_two(self, data_process_definition_id,
input_dataproduct, output_dataproduct):
# Create the data process
#data process 2 adds salinity + pressure and puts the result in conductivity
argument_map = {'arr1': 'salinity', 'arr2': 'pressure'}
output_param = 'conductivity'
dp2_func_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=data_process_definition_id,
inputs=[input_dataproduct],
outputs=[output_dataproduct],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp2_func_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
dp2_func_data_process_id)
self.dp_list.append(dp2_func_data_process_id)
return dp2_func_data_process_id
def create_output_data_products(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject(RT.DataProduct,
name='data_process1_data_product',
description='output of add array func')
dp1_func_output_dp_id = self.dataproductclient.create_data_product(
dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp1_func_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_one_id = stream_ids[0]
dp2_func_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp2_stream', parameter_dictionary_id=self.parameter_dict_id)
dp2_func_output_dp_obj = IonObject(
RT.DataProduct,
name='data_process2_data_product',
description='output of add array func')
dp2_func_output_dp_id = self.dataproductclient.create_data_product(
dp2_func_output_dp_obj, dp2_func_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp2_func_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp2_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_two_id = stream_ids[0]
subscription_id = self.pubsub_client.create_subscription(
'validator',
data_product_ids=[dp1_func_output_dp_id, dp2_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription,
subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s',
stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
validator = StandaloneStreamSubscriber('validator',
callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
subscription_id)
return dp1_func_output_dp_id, dp2_func_output_dp_id
def validate_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
data_process_event = args[0]
log.debug("DataProcessStatusEvent: %s",
str(data_process_event.__dict__))
#if data process already created, check origin
if not 'data process assigned to transform worker' in data_process_event.description:
self.assertIn(data_process_event.origin, self.dp_list)
def validate_output_granule(self, msg, route, stream_id):
self.assertTrue(
stream_id in
[self._output_stream_one_id, self._output_stream_two_id])
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule stream_id: %s', stream_id)
if stream_id == self._output_stream_one_id:
sal_val = rdt['salinity']
log.debug('validate_output_granule sal_val: %s', sal_val)
np.testing.assert_array_equal(sal_val, np.array([3]))
self.event1_verified.set()
else:
cond_val = rdt['conductivity']
log.debug('validate_output_granule cond_val: %s', cond_val)
np.testing.assert_array_equal(cond_val, np.array([5]))
self.event2_verified.set()
def start_event_listener(self):
es = EventSubscriber(event_type=OT.DataProcessStatusEvent,
callback=self.validate_event)
es.start()
self.addCleanup(es.stop)
class TestWorkflowManagementIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(
node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(
node=self.container.node)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def _create_ctd_input_stream_and_data_product(
self, data_product_name='ctd_parsed'):
cc = self.container
assertions = self.assertTrue
#-------------------------------
# Create CTD Parsed as the initial data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
container=self.ctd_stream_def, name='Simulated CTD data')
log.debug('Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name=data_product_name,
description='ctd stream test')
try:
ctd_parsed_data_product_id = self.dataproductclient.create_data_product(
dp_obj, ctd_stream_def_id)
except Exception as ex:
self.fail("failed to create new data product: %s" % ex)
log.debug('new ctd_parsed_data_product_id = ',
ctd_parsed_data_product_id)
#Only ever need one device for testing purposes.
instDevice_obj, _ = self.rrclient.find_resources(
restype=RT.InstrumentDevice, name='SBE37IMDevice')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_parsed_data_product_id,
persist_data=False,
persist_metadata=False)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id,
PRED.hasStream, None, True)
assertions(len(stream_ids) > 0)
ctd_stream_id = stream_ids[0]
return ctd_stream_id, ctd_parsed_data_product_id
def _start_simple_input_stream_process(self, ctd_stream_id):
return self._start_input_stream_process(ctd_stream_id)
def _start_sinusoidal_input_stream_process(self, ctd_stream_id):
return self._start_input_stream_process(
ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher',
'SinusoidalCtdPublisher')
def _start_input_stream_process(
self,
ctd_stream_id,
module='ion.processes.data.ctd_stream_publisher',
class_name='SimpleCtdPublisher'):
###
### Start the process for producing the CTD data
###
# process definition for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': module,
'class': class_name
}
ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(
process_definition=producer_definition)
# Start the ctd simulator to produce some data
configuration = {
'process': {
'stream_id': ctd_stream_id,
}
}
ctd_sim_pid = self.process_dispatcher.schedule_process(
process_definition_id=ctd_sim_procdef_id,
configuration=configuration)
return ctd_sim_pid
def _start_output_stream_listener(self,
data_product_stream_ids,
message_count_per_stream=10):
cc = self.container
assertions = self.assertTrue
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name='workflow_test',
name="test workflow transformations",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process,
node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn(' data received!')
results.append(message)
if len(results) >= len(
data_product_stream_ids
) * message_count_per_stream: #Only wait for so many messages - per stream
result.set(True)
subscriber = subscriber_registrar.create_subscriber(
exchange_name='workflow_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(
subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=30))
self.pubsubclient.deactivate_subscription(
subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def _validate_messages(self, results):
cc = self.container
assertions = self.assertTrue
first_salinity_values = None
for message in results:
try:
psd = PointSupplementStreamParser(
stream_definition=self.ctd_stream_def,
stream_granule=message)
temp = psd.get_values('temperature')
log.info(psd.list_field_names())
except KeyError as ke:
temp = None
if temp is not None:
assertions(isinstance(temp, numpy.ndarray))
log.info('temperature=' + str(numpy.nanmin(temp)))
first_salinity_values = None
else:
psd = PointSupplementStreamParser(
stream_definition=SalinityTransform.outgoing_stream_def,
stream_granule=message)
log.info(psd.list_field_names())
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
log.info('salinity=' + str(numpy.nanmin(salinity)))
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) >
0.0) # salinity should always be greater than 0
if first_salinity_values is None:
first_salinity_values = salinity.tolist()
else:
second_salinity_values = salinity.tolist()
assertions(
len(first_salinity_values) == len(
second_salinity_values))
for idx in range(0, len(first_salinity_values)):
assertions(first_salinity_values[idx] *
2.0 == second_salinity_values[idx])
def _create_salinity_data_process_definition(self):
# Salinity: Data Process Definition
#First look to see if it exists and if not, then create it
dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition,
name='ctd_salinity')
if len(dpd) > 0:
return dpd[0]
log.debug("Create data process definition SalinityTransform")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_salinity',
description='create a salinity data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except Excpetion as ex:
self.fail(
"failed to create new SalinityTransform data process definition: %s"
% ex)
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = self.pubsubclient.create_stream_definition(
container=SalinityTransform.outgoing_stream_def, name='Salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
sal_stream_def_id, ctd_L2_salinity_dprocdef_id)
return ctd_L2_salinity_dprocdef_id
def _create_salinity_doubler_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition,
name='salinity_doubler')
if len(dpd) > 0:
return dpd[0]
# Salinity Doubler: Data Process Definition
log.debug("Create data process definition SalinityDoublerTransform")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='salinity_doubler',
description='create a salinity doubler data product',
module='ion.processes.data.transforms.example_double_salinity',
class_name='SalinityDoubler',
process_source='SalinityDoubler source code here...')
try:
salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except Exception as ex:
self.fail(
"failed to create new SalinityDoubler data process definition: %s"
% ex)
# create a stream definition for the data from the salinity Transform
salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(
container=SalinityDoubler.outgoing_stream_def,
name='SalinityDoubler')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
salinity_double_stream_def_id, salinity_doubler_dprocdef_id)
return salinity_doubler_dprocdef_id
def create_transform_process(self, data_process_definition_id,
data_process_input_dp_id):
data_process_definition = self.rrclient.read(
data_process_definition_id)
# Find the link between the output Stream Definition resource and the Data Process Definition resource
stream_ids, _ = self.rrclient.find_objects(data_process_definition._id,
PRED.hasStreamDefinition,
RT.StreamDefinition,
id_only=True)
if not stream_ids:
raise Inconsistent(
"The data process definition %s is missing an association to an output stream definition"
% data_process_definition._id)
process_output_stream_def_id = stream_ids[0]
#Concatenate the name of the workflow and data process definition for the name of the data product output
data_process_name = data_process_definition.name
# Create the output data product of the transform
transform_dp_obj = IonObject(
RT.DataProduct,
name=data_process_name,
description=data_process_definition.description)
transform_dp_id = self.dataproductclient.create_data_product(
transform_dp_obj, process_output_stream_def_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=transform_dp_id,
persist_data=True,
persist_metadata=True)
#last one out of the for loop is the output product id
output_data_product_id = transform_dp_id
# Create the transform data process
log.debug("create data_process and start it")
data_process_id = self.dataprocessclient.create_data_process(
data_process_definition._id, [data_process_input_dp_id],
{'output': transform_dp_id})
self.dataprocessclient.activate_data_process(data_process_id)
#Find the id of the output data stream
stream_ids, _ = self.rrclient.find_objects(transform_dp_id,
PRED.hasStream, None, True)
if not stream_ids:
raise Inconsistent(
"The data process %s is missing an association to an output stream"
% data_process_id)
return data_process_id, output_data_product_id
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_SA_transform_components(self):
assertions = self.assertTrue
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
###
### Setup the first transformation
###
# Salinity: Data Process Definition
ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition(
)
l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process(
ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product_id)
## get the stream id for the transform outputs
stream_ids, _ = self.rrclient.find_objects(
ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0)
sal_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_stream_id)
###
### Setup the second transformation
###
# Salinity Doubler: Data Process Definition
salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition(
)
salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process(
salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(
salinity_doubler_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0)
sal_dbl_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_dbl_stream_id)
#Start the input stream process
ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id)
#Start te output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the transform processes
self.dataprocessclient.deactivate_data_process(
salinity_double_data_process_id)
self.dataprocessclient.deactivate_data_process(
l2_salinity_all_data_process_id)
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_messages(results)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='Salinity_Test_Workflow',
description='tests a workflow of multiple transform data processes'
)
workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name
#Add a transformation process definition
ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=ctd_L2_salinity_dprocdef_id,
persist_process_output_data=False
) #Don't persist the intermediate data product
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Add a transformation process definition
salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=salinity_doubler_dprocdef_id,
output_data_product_name=workflow_data_product_name)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
aids = self.rrclient.find_associations(workflow_def_id,
PRED.hasDataProcessDefinition)
assertions(len(aids) == 2)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id, timeout=30)
workflow_output_ids, _ = self.rrclient.find_subjects(
RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1)
#Verify the output data product name matches what was specified in the workflow definition
workflow_product = self.rrclient.read(workflow_product_id)
assertions(workflow_product.name == workflow_data_product_name)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids, _ = self.rrclient.find_objects(
workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 2)
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
None, True)
assertions(len(stream_ids) == 1)
data_product_stream_ids.append(stream_ids[0])
#Start the input stream process
ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id)
#Start the output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id, False, timeout=15) # Should test true at some point
#Make sure the Workflow object was removed
objs, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(objs) == 0)
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_messages(results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids, _ = self.rrclient.find_resources(
restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0)
def _create_google_dt_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition,
name='google_dt_transform')
if len(dpd) > 0:
return dpd[0]
# Data Process Definition
log.debug("Create data process definition GoogleDtTransform")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='google_dt_transform',
description='Convert data streams to Google DataTables',
module='ion.processes.data.transforms.viz.google_dt',
class_name='VizTransformGoogleDT',
process_source='VizTransformGoogleDT source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except Exception as ex:
self.fail(
"failed to create new VizTransformGoogleDT data process definition: %s"
% ex)
# create a stream definition for the data from the
stream_def_id = self.pubsubclient.create_stream_definition(
container=VizTransformGoogleDT.outgoing_stream_def,
name='VizTransformGoogleDT')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
stream_def_id, procdef_id)
return procdef_id
def _validate_google_dt_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g, Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
gdt_component = rdt['google_dt_components'][0]
assertions(
gdt_component['viz_product_type'] == 'google_realtime_dt')
gdt_description = gdt_component['data_table_description']
gdt_content = gdt_component['data_table_content']
assertions(gdt_description[0][0] == 'time')
assertions(len(gdt_description) > 1)
assertions(len(gdt_content) >= 0)
return
def _create_mpl_graphs_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition,
name='mpl_graphs_transform')
if len(dpd) > 0:
return dpd[0]
#Data Process Definition
log.debug("Create data process definition MatplotlibGraphsTransform")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='mpl_graphs_transform',
description='Convert data streams to Matplotlib graphs',
module='ion.processes.data.transforms.viz.matplotlib_graphs',
class_name='VizTransformMatplotlibGraphs',
process_source='VizTransformMatplotlibGraphs source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except Exception as ex:
self.fail(
"failed to create new VizTransformMatplotlibGraphs data process definition: %s"
% ex)
# create a stream definition for the data
stream_def_id = self.pubsubclient.create_stream_definition(
container=VizTransformMatplotlibGraphs.outgoing_stream_def,
name='VizTransformMatplotlibGraphs')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
stream_def_id, procdef_id)
return procdef_id
def _validate_mpl_graphs_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g, Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
graphs = rdt['matplotlib_graphs']
for graph in graphs:
assertions(
graph['viz_product_type'] == 'matplotlib_graphs')
# check to see if the list (numpy array) contians actual images
assertions(
imghdr.what(graph['image_name'], graph['image_obj']) ==
'png')
return
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#unittest.skip("Skipping for debugging ")
def test_google_dt_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='GoogleDT_Test_Workflow',
description=
'Tests the workflow of converting stream data to Google DT')
#Add a transformation process definition
google_dt_procdef_id = self._create_google_dt_data_process_definition()
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=google_dt_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id)
workflow_output_ids, _ = self.rrclient.find_subjects(
RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids, _ = self.rrclient.find_objects(
workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1)
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
None, True)
assertions(len(stream_ids) == 1)
data_product_stream_ids.append(stream_ids[0])
#Start the input stream process
ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id)
#Start the output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id, False) # Should test true at some point
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_google_dt_results(
VizTransformGoogleDT.outgoing_stream_def, results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids, _ = self.rrclient.find_resources(
restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_mpl_graphs_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='Mpl_Graphs_Test_Workflow',
description=
'Tests the workflow of converting stream data to Matplotlib graphs'
)
#Add a transformation process definition
mpl_graphs_procdef_id = self._create_mpl_graphs_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=mpl_graphs_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id)
workflow_output_ids, _ = self.rrclient.find_subjects(
RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids, _ = self.rrclient.find_objects(
workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1)
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
None, True)
assertions(len(stream_ids) == 1)
data_product_stream_ids.append(stream_ids[0])
#Start the input stream process
ctd_sim_pid = self._start_sinusoidal_input_stream_process(
ctd_stream_id)
#Start the output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id, False) # Should test true at some point
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_mpl_graphs_results(
VizTransformGoogleDT.outgoing_stream_def, results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids, _ = self.rrclient.find_resources(
restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_multiple_workflow_instances(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='Multiple_Test_Workflow',
description='Tests the workflow of converting stream data')
#Add a transformation process definition
google_dt_procdef_id = self._create_google_dt_data_process_definition()
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=google_dt_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the first input data product
ctd_stream_id1, ctd_parsed_data_product_id1 = self._create_ctd_input_stream_and_data_product(
'ctd_parsed1')
data_product_stream_ids.append(ctd_stream_id1)
#Create and start the first workflow
workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id1)
#Create the second input data product
ctd_stream_id2, ctd_parsed_data_product_id2 = self._create_ctd_input_stream_and_data_product(
'ctd_parsed2')
data_product_stream_ids.append(ctd_stream_id2)
#Create and start the first workflow
workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id2)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 2)
#Start the first input stream process
ctd_sim_pid1 = self._start_sinusoidal_input_stream_process(
ctd_stream_id1)
#Start the second input stream process
ctd_sim_pid2 = self._start_simple_input_stream_process(ctd_stream_id2)
#Start the output stream listener to monitor a set number of messages being sent through the workflows
results = self._start_output_stream_listener(
data_product_stream_ids, message_count_per_stream=5)
# stop the flow of messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid1
) # kill the ctd simulator process - that is enough data
self.process_dispatcher.cancel_process(ctd_sim_pid2)
#Stop the first workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id1, False) # Should test true at some point
#Stop the second workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id2, False) # Should test true at some point
workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 0)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids, _ = self.rrclient.find_resources(
restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0)
aid_list = self.rrclient.find_associations(
workflow_def_id, PRED.hasDataProcessDefinition)
assertions(len(aid_list) == 0)
class TestDataProcessWithLookupTable(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
self.processdispatchclient = ProcessDispatcherServiceClient(node = self.container.node)
def test_lookupTableProcessing(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.info('test_createTransformsThenActivateInstrument: new InstrumentModel id = %s', instModel_id)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri=DRV_URI_GOOD)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.info( 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice and attachment for lookup table
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.info( 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s', instDevice_id)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(instDevice_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance" )
self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id)
log.info( 'TestDataProcessWithLookupTable: new Stream Definition id = %s', instDevice_id)
log.info( 'Creating new CDM data product with a stream definition')
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
log.info( 'new ctd_parsed_data_product_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
log.info('TestDataProcessWithLookupTable: Data product streams1 = %s', stream_ids)
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
log.info('TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition')
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='ctd_raw',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
log.info( 'new ctd_raw_data_product_id = %s', ctd_raw_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
log.info( 'Data product streams2 = %s', stream_ids)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..."
att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) )
processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) )
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProcessWithLookupTable: create L0 all data_process start")
try:
in_prods = []
in_prods.append(ctd_parsed_data_product)
out_prods = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id,ctd_l0_temperature_output_dp_id]
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L0_all_dprocdef_id,
in_data_product_ids = in_prods,
out_data_product_ids = out_prods)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("TestDataProcessWithLookupTable: create L0 all data_process return")
data_process= self.rrclient.read(ctd_l0_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(subject= ctd_l0_all_data_process_id, predicate= PRED.hasProcess, object_type= RT.Process,id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process, process_ids[0])
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..."
att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s', processAttachment)
class TestDataProcessWithLookupTable(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
#@unittest.skip('not working')
def test_lookupTableProcessing(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.services.mi.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice and attachment for lookup table
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(instDevice_id, att)
print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', deviceAttachment
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
driver_config = {
'dvr_mod' : 'ion.services.mi.drivers.sbe37_driver',
'dvr_cls' : 'SBE37Driver',
'workdir' : '/tmp/',
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test')
try:
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..."
att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) )
processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) )
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity')
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure')
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature')
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, ctd_parsed_data_product, self.output_products)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..."
att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', processAttachment
class CtdbpTransformsIntTest(IonIntegrationTestCase):
def setUp(self):
super(CtdbpTransformsIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.dataproduct_management = DataProductManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
# This is for the time values inside the packets going into the transform
self.i = 0
# Cleanup of queue created by the subscriber
def _get_new_ctd_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
rdt['time'] = numpy.arange(self.i, self.i+length)
for field in rdt:
if isinstance(rdt._pdict.get_context(field).param_type, QuantityType):
rdt[field] = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)])
g = rdt.to_granule()
self.i+=length
return g
def _create_input_param_dict_for_test(self, parameter_dict_name = ''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = ''
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = ''
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = ''
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = ''
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = ''
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
def test_ctdbp_L0_all(self):
"""
Test packets processed by the ctdbp_L0_all transform
"""
#----------- Data Process Definition --------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='CTDBP_L0_all',
description='Take parsed stream and put the C, T and P into three separate L0 streams.',
module='ion.processes.data.transforms.ctdbp.ctdbp_L0',
class_name='CTDBP_L0_all')
dprocdef_id = self.data_process_management.create_data_process_definition(dpd_obj)
self.addCleanup(self.data_process_management.delete_data_process_definition, dprocdef_id)
log.debug("created data process definition: id = %s", dprocdef_id)
#----------- Data Products --------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_param_dict = self._create_input_param_dict_for_test(parameter_dict_name = 'fictitious_ctdp_param_dict')
# Get the stream definition for the stream using the parameter dictionary
# input_param_dict = self.dataset_management.read_parameter_dictionary_by_name('ctdbp_cdef_sample', id_only=True)
input_stream_def_dict = self.pubsub.create_stream_definition(name='parsed', parameter_dictionary_id=input_param_dict)
self.addCleanup(self.pubsub.delete_stream_definition, input_stream_def_dict)
log.debug("Got the parsed parameter dictionary: id: %s", input_param_dict)
log.debug("Got the stream def for parsed input: %s", input_stream_def_dict)
# Input data product
parsed_stream_dp_obj = IonObject(RT.DataProduct,
name='parsed_stream',
description='Parsed stream input to CTBP L0 transform',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproduct_management.create_data_product(data_product=parsed_stream_dp_obj,
stream_definition_id=input_stream_def_dict
)
self.addCleanup(self.dataproduct_management.delete_data_product, input_dp_id)
# output data product
L0_stream_dp_obj = IonObject(RT.DataProduct,
name='L0_stream',
description='L0_stream output of CTBP L0 transform',
temporal_domain = tdom,
spatial_domain = sdom)
L0_stream_dp_id = self.dataproduct_management.create_data_product(data_product=L0_stream_dp_obj,
stream_definition_id=input_stream_def_dict
)
self.addCleanup(self.dataproduct_management.delete_data_product, L0_stream_dp_id)
# We need the key name here to be "L0_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L0_stream when the config is used to launch the data process
self.output_products = {'L0_stream' : L0_stream_dp_id}
out_stream_ids, _ = self.resource_registry.find_objects(L0_stream_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(out_stream_ids))
output_stream_id = out_stream_ids[0]
dproc_id = self.data_process_management.create_data_process( dprocdef_id, [input_dp_id], self.output_products)
self.addCleanup(self.data_process_management.delete_data_process, dproc_id)
log.debug("Created a data process for ctdbp_L0. id: %s", dproc_id)
# Activate the data process
self.data_process_management.activate_data_process(dproc_id)
self.addCleanup(self.data_process_management.deactivate_data_process, dproc_id)
#----------- Find the stream that is associated with the input data product when it was created by create_data_product() --------------------------------
stream_ids, _ = self.resource_registry.find_objects(input_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(stream_ids))
input_stream_id = stream_ids[0]
stream_route = self.pubsub.read_stream_route(input_stream_id)
log.debug("The input stream for the L0 transform: %s", input_stream_id)
#----------- Create a subscriber that will listen to the transform's output --------------------------------
ar = gevent.event.AsyncResult()
def subscriber(m,r,s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub', callback=subscriber)
sub_id = self.pubsub.create_subscription('subscriber_to_transform',
stream_ids=[output_stream_id],
exchange_name='sub')
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
#----------- Publish on that stream so that the transform can receive it --------------------------------
pub = StandaloneStreamPublisher(input_stream_id, stream_route)
publish_granule = self._get_new_ctd_packet(stream_definition_id=input_stream_def_dict, length = 5)
pub.publish(publish_granule)
log.debug("Published the following granule: %s", publish_granule)
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the transform: %s", granule_from_transform)
# Check that the granule published by the L0 transform has the right properties
self._check_granule_from_transform(granule_from_transform)
def _check_granule_from_transform(self, granule):
"""
An internal method to check if a granule has the right properties
"""
pass
class TestCTDTransformsIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
def test_createTransformsThenActivateInstrument(self):
# Set up the preconditions
# ingestion configuration parameters
self.exchange_point_id = 'science_data'
self.number_of_workers = 2
self.hdf_storage = HdfStorage(relative_path='ingest')
self.couch_storage = CouchStorage(datastore_name='test_datastore')
self.XP = 'science_data'
self.exchange_name = 'ingestion_queue'
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.services.mi.instrument_agent", driver_class="InstrumentAgent")
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
# driver_config = {
# 'svr_addr': "localhost",
# 'cmd_port': 5556,
# 'evt_port': 5557,
# 'dvr_mod': "ion.services.mi.drivers.sbe37_driver",
# 'dvr_cls': "SBE37Driver",
# 'comms_config': {
# 'addr': 'sbe37-simulator.oceanobservatories.org',
# 'port': 4001,
# }
# }
driver_config = {
'dvr_mod' : 'ion.services.mi.drivers.sbe37_driver',
'dvr_cls' : 'SBE37Driver',
'workdir' : '/tmp/',
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test')
try:
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity')
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure')
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature')
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id )
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id )
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L1_Conductivity',description='transform output L1 conductivity')
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L1_Pressure',description='transform output L1 pressure')
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L1_Temperature',description='transform output L1 temperature')
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id )
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity',description='transform output L2 salinity')
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density")
ctd_l2_density_output_dp_obj = IonObject(RT.DataProduct, name='L2_Density',description='transform output pressure')
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, ctd_parsed_data_product, self.output_products)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, ctd_l0_conductivity_output_dp_id, {'output':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return")
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, ctd_l0_pressure_output_dp_id, {'output':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, ctd_l0_temperature_output_dp_id, {'output':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product, {'output':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, ctd_parsed_data_product, {'output':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return")
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Streaming
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
print retval
log.debug("test_createTransformsThenActivateInstrument:: initialize %s", str(retval))
time.sleep(1)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: go_active %s", str(reply))
time.sleep(1)
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: run %s", str(reply))
time.sleep(1)
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG : 1,
SBE37Parameter.INTERVAL : 5,
SBE37Parameter.TXREALTIME : True
}
self._ia_client.set_param(params)
self._no_samples = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: go_streaming %s", str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(15)
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrument: return from go_observatory %s", str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
log.debug("test_createTransformsThenActivateInstrument:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: return from reset %s", str(reply))
time.sleep(2)
#-------------------------------
# Sampling
#-------------------------------
# cmd = AgentCommand(command='initialize')
# retval = self._ia_client.execute_agent(cmd)
# print retval
# log.debug("test_createTransformsThenActivateInstrument:: initialize %s", str(retval))
# time.sleep(2)
#
# cmd = AgentCommand(command='go_active')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: go_active %s", str(reply))
# time.sleep(2)
#
# cmd = AgentCommand(command='run')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: run %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample ")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 2")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 2 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 3")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 3 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling go_inactive ")
# cmd = AgentCommand(command='go_inactive')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from go_inactive %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling reset ")
# cmd = AgentCommand(command='reset')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from reset %s", str(reply))
# time.sleep(2)
#
#
# self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
#
#
# #get the dataset id of the ctd_parsed product from the dataproduct ctd_parsed_data_product
# ctd_parsed_data_product_obj = self.dataproductclient.read_data_product(ctd_parsed_data_product)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset id %s", str(ctd_parsed_data_product_obj.dataset_id))
#
# # ask for the dataset bounds from the datasetmgmtsvc
# bounds = self.datasetclient.get_dataset_bounds(ctd_parsed_data_product_obj.dataset_id)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset bounds %s", str(bounds))
# print 'activate_instrument: got dataset bounds %s', str(bounds)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
class TestCTDTransformsNoSim(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
@unittest.skip('test not working')
def test_createTransformsThenPublishGranules(self):
# ctd simulator process
producer_definition = ProcessDefinition(name='Example Data Producer')
producer_definition.executable = {
'module':'ion.services.sa.test.simple_ctd_data_producer',
'class':'SimpleCtdDataProducer'
}
producer_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', ctd_stream_def_id
print 'Creating new CDM data product with a stream definition'
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
#self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product))
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
self.parsed_stream_id = stream_ids[0]
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='ctd_raw',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
#self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
self.loggerpids = []
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id,
parameter_dictionary)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id )
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id )
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l1_conductivity stream id = %s", str(stream_ids) )
pid = self.create_logger(' ctd_l1_conductivity', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l1_pressure stream id = %s", str(stream_ids) )
pid = self.create_logger(' ctd_l1_pressure', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l1_temperature stream id = %s", str(stream_ids) )
pid = self.create_logger(' ctd_l1_temperature', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id )
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density")
ctd_l2_density_output_dp_obj = IonObject(RT.DataProduct,
name='L2_Density',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id)
# Set up subscribers/loggers to these streams
stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
log.debug("L2 salinity stream id = %s", str(stream_ids) )
pid = self.create_logger('L2_salinity', stream_ids[0] )
self.loggerpids.append(pid)
stream_ids, _ = self.rrclient.find_objects(ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
log.debug("L2 density stream id = %s", str(stream_ids) )
pid = self.create_logger('L2_density', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return")
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return")
#-------------------------------
# Streaming
#-------------------------------
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':self.parsed_stream_id,
}
}
producer_pid = self.processdispatchclient.schedule_process(process_definition_id= producer_procdef_id, configuration=configuration)
time.sleep(2.0)
# clean up the launched processes
self.processdispatchclient.cancel_process(producer_pid)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestDataProcessWithLookupTable(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
def test_lookupTableProcessing(self):
# -------------------------------
# Create InstrumentModel
# -------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
log.info("test_createTransformsThenActivateInstrument: new InstrumentModel id = %s", instModel_id)
# -------------------------------
# Create InstrumentAgent
# -------------------------------
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.info("test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s", instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# -------------------------------
# Create InstrumentDevice and attachment for lookup table
# -------------------------------
instDevice_obj = IonObject(
RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345"
)
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.info("test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s", instDevice_id)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(
RT.Attachment,
name="deviceLookupTable",
content=base64.encodestring(contents),
keywords=["DataProcessInput"],
attachment_type=AttachmentType.ASCII,
)
deviceAttachment = self.rrclient.create_attachment(instDevice_id, att)
log.info("test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s", deviceAttachment)
# -------------------------------
# Create InstrumentAgentInstance to hold configuration information
# -------------------------------
driver_config = {
"dvr_mod": "mi.instrument.seabird.sbe37smb.ooicore.driver",
"dvr_cls": "SBE37Driver",
"workdir": "/tmp/",
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstance",
description="SBE37IMAgentInstance",
driver_config=driver_config,
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4001,
port_agent_work_dir="/tmp/",
port_agent_delimeter=["<<", ">>"],
)
self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
# -------------------------------
# Create CTD Parsed as the first data product
# -------------------------------
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
name="SBE37_CDM", parameter_dictionary_id=pdict_id
)
log.info("TestDataProcessWithLookupTable: new Stream Definition id = %s", instDevice_id)
log.info("Creating new CDM data product with a stream definition")
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(
RT.DataProduct, name="ctd_parsed", description="ctd stream test", temporal_domain=tdom, spatial_domain=sdom
)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
log.info("new ctd_parsed_data_product_id = %s", ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
log.info("TestDataProcessWithLookupTable: Data product streams1 = %s", stream_ids)
# -------------------------------
# Create CTD Raw as the second data product
# -------------------------------
log.info("TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition")
raw_stream_def_id = self.pubsubclient.create_stream_definition(
name="SBE37_RAW", parameter_dictionary_id=pdict_id
)
dp_obj = IonObject(
RT.DataProduct, name="ctd_raw", description="raw stream test", temporal_domain=tdom, spatial_domain=sdom
)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
log.info("new ctd_raw_data_product_id = %s", ctd_raw_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
log.info("Data product streams2 = %s", stream_ids)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
# -------------------------------
log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L0_all",
description="transform ctd package into three separate L0 streams",
module="ion.processes.data.transforms.ctd.ctd_L0_all",
class_name="ctd_L0_all",
)
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" % ex)
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..."
att = IonObject(
RT.Attachment,
name="processDefinitionLookupTable",
content=base64.encodestring(contents),
keywords=["DataProcessInput"],
attachment_type=AttachmentType.ASCII,
)
processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att)
log.debug(
"TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s",
str(processDefinitionAttachment),
)
processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment)
log.debug(
"TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s",
str(processDefinitionAttachment_obj),
)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
# -------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name="L0_Conductivity", parameter_dictionary_id=pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding="conductivity"
)
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name="L0_Pressure", parameter_dictionary_id=pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding="pressure"
)
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name="L0_Temperature", parameter_dictionary_id=pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding="temperature"
)
self.output_products = {}
log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Conductivity",
description="transform output conductivity",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id
)
self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id
log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Pressure",
description="transform output pressure",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id
)
self.output_products["pressure"] = ctd_l0_pressure_output_dp_id
log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Temperature",
description="transform output temperature",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id
)
self.output_products["temperature"] = ctd_l0_temperature_output_dp_id
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
# -------------------------------
log.debug("TestDataProcessWithLookupTable: create L0 all data_process start")
try:
in_prods = []
in_prods.append(ctd_parsed_data_product)
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, in_prods, self.output_products
)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug("TestDataProcessWithLookupTable: create L0 all data_process return")
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..."
att = IonObject(
RT.Attachment,
name="processLookupTable",
content=base64.encodestring(contents),
keywords=["DataProcessInput"],
attachment_type=AttachmentType.ASCII,
)
processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info("TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s", processAttachment)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
def test_createDataProcess(self):
#---------------------------------------------------------------------------
# Data Process Definition
#---------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# Make assertion on the newly registered data process definition
data_process_definition = self.rrclient.read(dprocdef_id)
self.assertEquals(data_process_definition.name, 'ctd_L0_all')
self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams')
self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all')
self.assertEquals(data_process_definition.class_name, 'ctd_L0_all')
# Read the data process definition using data process management and make assertions
dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id)
self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all')
self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all')
#---------------------------------------------------------------------------
# Create an input instrument
#---------------------------------------------------------------------------
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
# Register the instrument so that the data producer and stream object are created
data_producer_id = self.damsclient.register_instrument(instrument_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id )
# Assert that the link between the stream definition and the data process definition was done
assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True)
self.assertIsNotNone(assocs)
#---------------------------------------------------------------------------
# Input Data Product
#---------------------------------------------------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_dp_obj = IonObject( RT.DataProduct,
name='InputDataProduct',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test')
#Make assertions on the input data product created
input_dp_obj = self.rrclient.read(input_dp_id)
self.assertEquals(input_dp_obj.name, 'InputDataProduct')
self.assertEquals(input_dp_obj.description, 'some new dp')
self.damsclient.assign_data_product(instrument_id, input_dp_id)
# Retrieve the stream via the DataProduct->Stream associations
stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True)
self.in_stream_id = stream_ids[0]
#---------------------------------------------------------------------------
# Output Data Product
#---------------------------------------------------------------------------
outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' )
outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' )
outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' )
self.output_products={}
output_dp_obj = IonObject(RT.DataProduct,
name='conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id)
self.output_products['conductivity'] = output_dp_id_1
output_dp_obj = IonObject(RT.DataProduct,
name='pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id)
self.output_products['pressure'] = output_dp_id_2
output_dp_obj = IonObject(RT.DataProduct,
name='temperature',
description='transform output ',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id)
self.output_products['temperature'] = output_dp_id_3
#---------------------------------------------------------------------------
# Create the data process
#---------------------------------------------------------------------------
def _create_data_process():
dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products)
return dproc_id
dproc_id = _create_data_process()
# Make assertions on the data process created
data_process = self.dataprocessclient.read_data_process(dproc_id)
# Assert that the data process has a process id attached
self.assertIsNotNone(data_process.process_id)
# Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute
self.assertIsNotNone(data_process.input_subscription_id)
output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True)
self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3]))
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}})
def test_createDataProcessUsingSim(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" )
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg")
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4002,
'data_port': 4003,
'log_level': 5,
}
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id)
dp_obj.name = 'ctd_raw'
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# Create listener for data process events and verify that events are received.
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events
#todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
data_process = self.rrclient.read(ctd_l0_all_data_process_id)
process_id = data_process.process_id
self.addCleanup(self.process_dispatcher.cancel_process, process_id)
#-------------------------------
# Wait until the process launched in the create_data_process() method is actually running, before proceeding further in this test
#-------------------------------
gate = ProcessStateGate(self.process_dispatcher.read_process, process_id, ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The data process (%s) did not spawn in 30 seconds" % process_id)
#-------------------------------
# Retrieve a list of all data process defintions in RR and validate that the DPD is listed
#-------------------------------
# todo: Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions
# todo: data topics are being handled by pub sub at the level of streams
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
#todo: check that activate event is received L4-CI-SA-RQ-367
#todo... (it looks like no event is being published when the data process is activated... so below, we just check for now
# todo... that the subscription is indeed activated) (Swarbhanu)
# todo: monitor process to see if it is active (sa-rq-182)
ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id)
input_subscription_id = ctd_l0_all_data_process.input_subscription_id
subs = self.rrclient.read(input_subscription_id)
self.assertTrue(subs.activated)
# todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the extended resources for data process definition and for data process
#-------------------------------
extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
self.assertEqual(1, len(extended_process_definition.data_processes))
log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition))
extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id)
self.assertEqual(1, len(extended_process.input_data_products))
log.debug("test_createDataProcess: extended_process %s", str(extended_process))
################################ Test the removal of data processes ##################################
#-------------------------------------------------------------------
# Test the deleting of the data process
#-------------------------------------------------------------------
# Before deleting, get the input streams, output streams and the subscriptions so that they can be checked after deleting
# dp_obj_1 = self.rrclient.read(ctd_l0_all_data_process_id)
# input_subscription_id = dp_obj_1.input_subscription_id
# out_prods, _ = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True)
# in_prods, _ = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasInputProduct, id_only=True)
# in_streams = []
# for in_prod in in_prods:
# streams, _ = self.rrclient.find_objects(in_prod, PRED.hasStream, id_only=True)
# in_streams.extend(streams)
# out_streams = []
# for out_prod in out_prods:
# streams, _ = self.rrclient.find_objects(out_prod, PRED.hasStream, id_only=True)
# out_streams.extend(streams)
# Deleting the data process
self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id)
# Check that the data process got removed. Check the lcs state. It should be retired
dp_obj = self.rrclient.read(ctd_l0_all_data_process_id)
self.assertEquals(dp_obj.lcstate, LCS.RETIRED)
# Check for process defs still attached to the data process
dpd_assn_ids = self.rrclient.find_associations(subject=ctd_l0_all_data_process_id, predicate=PRED.hasProcessDefinition, id_only=True)
self.assertEquals(len(dpd_assn_ids), 0)
# Check for output data product still attached to the data process
out_products, assocs = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, predicate=PRED.hasOutputProduct, id_only=True)
self.assertEquals(len(out_products), 0)
self.assertEquals(len(assocs), 0)
# Check for input data products still attached to the data process
inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct)
self.assertEquals(len(inprod_associations), 0)
# Check for input data products still attached to the data process
inprod_associations = self.rrclient.find_associations(ctd_l0_all_data_process_id, PRED.hasInputProduct)
self.assertEquals(len(inprod_associations), 0)
# Check of the data process has been deactivated
self.assertIsNone(dp_obj.input_subscription_id)
# Read the original subscription id of the data process and check that it has been deactivated
with self.assertRaises(NotFound):
self.pubsubclient.read_subscription(input_subscription_id)
#-------------------------------------------------------------------
# Delete the data process definition
#-------------------------------------------------------------------
# before deleting, get the process definition being associated to in order to be able to check later if the latter gets deleted as it should
proc_def_ids, proc_def_asocs = self.rrclient.find_objects(ctd_l0_all_data_process_id, PRED.hasProcessDefinition)
self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id)
# check that the data process definition has been retired
dp_proc_def = self.rrclient.read(ctd_L0_all_dprocdef_id)
self.assertEquals(dp_proc_def.lcstate, LCS.RETIRED)
# Check for old associations of this data process definition
proc_defs, proc_def_asocs = self.rrclient.find_objects(ctd_L0_all_dprocdef_id, PRED.hasProcessDefinition)
self.assertEquals(len(proc_defs), 0)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject= ctd_L0_all_dprocdef_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
################################ Test the removal of data processes ##################################
# Try force delete... This should simply delete the associations and the data process object
# from the resource registry
#---------------------------------------------------------------------------------------------------------------
# Force deleting a data process
#---------------------------------------------------------------------------------------------------------------
self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True)
# find all associations where this is the object
_, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
self.assertEquals(len(sbj_assns), 0)
with self.assertRaises(NotFound):
self.rrclient.read(ctd_l0_all_data_process_id)
#---------------------------------------------------------------------------------------------------------------
# Force deleting a data process definition
#---------------------------------------------------------------------------------------------------------------
self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id)
# find all associations where this is the subject
_, obj_assns = self.rrclient.find_objects(subject=ctd_l0_all_data_process_id, id_only=True)
# find all associations where this is the object
_, sbj_assns = self.rrclient.find_subjects(object=ctd_l0_all_data_process_id, id_only=True)
self.assertEquals(len(obj_assns), 0)
self.assertEquals(len(sbj_assns), 0)
with self.assertRaises(NotFound):
self.rrclient.read(ctd_l0_all_data_process_id)
class TestIntDataProcessManagementService(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
# Establish endpoint with container
container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
#print 'got CC client'
container_client.start_rel_from_url('res/deploy/r2sa.yml')
# Now create client to DataProcessManagementService
self.Processclient = DataProcessManagementServiceClient(node=self.container.node)
self.RRclient = ResourceRegistryServiceClient(node=self.container.node)
self.DAMSclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DPMSclient = DataProductManagementServiceClient(node=self.container.node)
self.PubSubClient = PubsubManagementServiceClient(node=self.container.node)
def test_createDataProcess(self):
#-------------------------------
# Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessManagementService: create data process definition")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='data_process_definition',
description='some new dpd',
module='ion.processes.data.transforms.transform_example',
class_name='TransformExample',
process_source='some_source_reference')
try:
dprocdef_id = self.Processclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new data process definition: %s" %ex)
# test Data Process Definition creation in rr
dprocdef_obj = self.Processclient.read_data_process_definition(dprocdef_id)
self.assertEquals(dprocdef_obj.name,'data_process_definition')
# Create an input instrument
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.RRclient.create(instrument_obj)
# Register the instrument so that the data producer and stream object are created
data_producer_id = self.DAMSclient.register_instrument(instrument_id)
log.debug("TestIntDataProcessManagementService data_producer_id %s" % data_producer_id)
#-------------------------------
# Input Data Product
#-------------------------------
log.debug("TestIntDataProcessManagementService: create input data product")
input_dp_obj = IonObject(RT.DataProduct, name='InputDataProduct', description='some new dp')
try:
input_dp_id = self.DPMSclient.create_data_product(input_dp_obj, instrument_id)
except BadRequest as ex:
self.fail("failed to create new input data product: %s" %ex)
# Retrieve the stream via the DataProduct->Stream associations
stream_ids, _ = self.RRclient.find_objects(input_dp_id, PRED.hasStream, None, True)
log.debug("TestIntDataProcessManagementService: in stream_ids " + str(stream_ids))
self.in_stream_id = stream_ids[0]
log.debug("TestIntDataProcessManagementService: Input Stream: " + str( self.in_stream_id))
#-------------------------------
# Output Data Product
#-------------------------------
log.debug("TestIntDataProcessManagementService: create output data product")
output_dp_obj = IonObject(RT.DataProduct, name='OutDataProduct',description='transform output')
output_dp_id = self.DPMSclient.create_data_product(output_dp_obj)
# this will NOT create a stream for the product becuase the data process (source) resource has not been created yet.
#-------------------------------
# Create the data process
#-------------------------------
log.debug("TestIntDataProcessManagementService: create_data_process start")
try:
dproc_id = self.Processclient.create_data_process(dprocdef_id, input_dp_id, output_dp_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
self.DAMSclient.assign_data_product(dproc_id, output_dp_id, False)
log.debug("TestIntDataProcessManagementService: create_data_process return")
#-------------------------------
# Producer (Sample Input)
#-------------------------------
# Create a producing example process
# cheat to make a publisher object to send messages in the test.
# it is really hokey to pass process=self.cc but it works
#stream_route = self.PubSubClient.register_producer(exchange_name='producer_doesnt_have_a_name1', stream_id=self.in_stream_id)
#self.ctd_stream1_publisher = StreamPublisher(node=self.container.node, name=('science_data',stream_route.routing_key), process=self.container)
pid = self.container.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = self.container.proc_manager.procs[pid]
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.container.node)
self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.in_stream_id)
msg = {'num':'3'}
self.ctd_stream1_publisher.publish(msg)
time.sleep(1)
msg = {'num':'5'}
self.ctd_stream1_publisher.publish(msg)
time.sleep(1)
msg = {'num':'9'}
self.ctd_stream1_publisher.publish(msg)
# See /tmp/transform_output for results.....
# clean up the data process
try:
self.Processclient.delete_data_process(dproc_id)
except BadRequest as ex:
self.fail("failed to create new data process definition: %s" %ex)
with self.assertRaises(NotFound) as e:
self.Processclient.read_data_process(dproc_id)
try:
self.Processclient.delete_data_process_definition(dprocdef_id)
except BadRequest as ex:
self.fail("failed to create new data process definition: %s" %ex)
with self.assertRaises(NotFound) as e:
self.Processclient.read_data_process_definition(dprocdef_id)
class TestIMSDeployAsPrimaryDevice(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
# self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
print "started services"
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def create_logger(self, name, stream_id=""):
# logger process
producer_definition = ProcessDefinition(name=name + "_logger")
producer_definition.executable = {
"module": "ion.processes.data.stream_granule_logger",
"class": "StreamGranuleLogger",
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {"process": {"stream_id": stream_id}}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id, configuration=configuration
)
return pid
def cleanupprocs(self):
stm = os.popen("ps -e | grep ion.agents.port.logger_process")
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
stm = os.popen("ps -e | grep ion.agents.instrument.zmq_driver_process")
procs = stm.read()
if len(procs) > 0:
procs = procs.split()
if procs[0].isdigit():
pid = int(procs[0])
os.kill(pid, signal.SIGKILL)
# stm = os.popen('rm /tmp/*.pid.txt')
@unittest.skip("Deprecated by IngestionManagement refactor, timeout on start inst agent?")
def test_deploy_activate_full(self):
# ensure no processes or pids are left around by agents or Sims
# self.cleanupprocs()
self.loggerpids = []
# -------------------------------
# Create InstrumentModel
# -------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
# -------------------------------
# Create InstrumentAgent
# -------------------------------
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug("new InstrumentAgent id = %s", instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# -------------------------------
# Create Instrument Site
# -------------------------------
instrumentSite_obj = IonObject(RT.InstrumentSite, name="instrumentSite1", description="SBE37IMInstrumentSite")
try:
instrumentSite_id = self.omsclient.create_instrument_site(instrument_site=instrumentSite_obj, parent_id="")
except BadRequest as ex:
self.fail("failed to create new InstrumentSite: %s" % ex)
print "test_deployAsPrimaryDevice: new instrumentSite id = ", instrumentSite_id
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrumentSite_id)
# -------------------------------
# Logical Transform: Output Data Products
# -------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
"ctd_parsed_param_dict", id_only=True
)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(
name="parsed", parameter_dictionary_id=parsed_pdict_id
)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_raw_param_dict", id_only=True)
raw_stream_def_id = self.pubsubclient.create_stream_definition(name="raw", parameter_dictionary_id=raw_pdict_id)
# -------------------------------
# Create Old InstrumentDevice
# -------------------------------
instDevice_obj = IonObject(
RT.InstrumentDevice,
name="SBE37IMDeviceYear1",
description="SBE37IMDevice for the FIRST year of deployment",
serial_number="12345",
)
try:
oldInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, oldInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print "test_deployAsPrimaryDevice: new Year 1 InstrumentDevice id = ", oldInstDevice_id
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(oldInstDevice_id, LCE.ENABLE)
# -------------------------------
# Create Raw and Parsed Data Products for the device
# -------------------------------
dp_obj = IonObject(
RT.DataProduct,
name="SiteDataProduct",
description="SiteDataProduct",
temporal_domain=tdom,
spatial_domain=sdom,
)
instrument_site_output_dp_id = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id
)
self.damsclient.assign_data_product(
input_resource_id=oldInstDevice_id, data_product_id=instrument_site_output_dp_id
)
# self.dataproductclient.activate_data_product_persistence(data_product_id=instrument_site_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasStream, None, True)
log.debug("Data product streams1 = %s", stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(instrument_site_output_dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug("Data set for data_product_id1 = %s", dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
self.omsclient.create_site_data_product(instrumentSite_id, instrument_site_output_dp_id)
# -------------------------------
# Create Old Deployment
# -------------------------------
deployment_obj = IonObject(RT.Deployment, name="first deployment")
oldDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(oldInstDevice_id, oldDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id, oldDeployment_id)
# -------------------------------
# Create InstrumentAgentInstance for OldInstrumentDevice to hold configuration information
# cmd_port=5556, evt_port=5557, comms_method="ethernet", comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
# -------------------------------
port_agent_config = {
"device_addr": CFG.device.sbe37.host,
"device_port": CFG.device.sbe37.port,
"process_type": PortAgentProcessType.UNIX,
"binary_path": "port_agent",
"port_agent_addr": "localhost",
"command_port": CFG.device.sbe37.port_agent_cmd_port,
"data_port": CFG.device.sbe37.port_agent_data_port,
"log_level": 5,
"type": PortAgentType.ETHERNET,
}
raw_config = StreamConfiguration(
stream_name="raw",
parameter_dictionary_name="ctd_raw_param_dict",
records_per_granule=2,
granule_publish_rate=5,
)
parsed_config = StreamConfiguration(
stream_name="parsed",
parameter_dictionary_name="ctd_parsed_param_dict",
records_per_granule=2,
granule_publish_rate=5,
)
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstanceYear1",
description="SBE37IMAgentInstanceYear1",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4001,
port_agent_config=port_agent_config,
stream_configurations=[raw_config, parsed_config],
)
oldInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, oldInstDevice_id
)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
# -------------------------------
# Create CTD Parsed as the Year 1 data product and attach to instrument
# -------------------------------
print "Creating new CDM data product with a stream definition"
dp_obj = IonObject(
RT.DataProduct,
name="ctd_parsed_year1",
description="ctd stream test year 1",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_parsed_data_product_year1 = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id
)
print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year1
self.damsclient.assign_data_product(
input_resource_id=oldInstDevice_id, data_product_id=ctd_parsed_data_product_year1
)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year1, PRED.hasStream, None, True)
print "test_deployAsPrimaryDevice: Data product streams1 = ", stream_ids
# -------------------------------
# Create New InstrumentDevice
# -------------------------------
instDevice_obj_2 = IonObject(
RT.InstrumentDevice,
name="SBE37IMDeviceYear2",
description="SBE37IMDevice for the SECOND year of deployment",
serial_number="67890",
)
try:
newInstDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj_2)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, newInstDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print "test_deployAsPrimaryDevice: new Year 2 InstrumentDevice id = ", newInstDevice_id
# set the LCSTATE
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.DEPLOY)
self.rrclient.execute_lifecycle_transition(newInstDevice_id, LCE.ENABLE)
instDevice_obj_2 = self.rrclient.read(newInstDevice_id)
log.debug("test_deployAsPrimaryDevice: Create New InstrumentDevice LCSTATE: %s ", str(instDevice_obj_2.lcstate))
# -------------------------------
# Create Old Deployment
# -------------------------------
deployment_obj = IonObject(RT.Deployment, name="second deployment")
newDeployment_id = self.omsclient.create_deployment(deployment_obj)
# deploy this device to the logical slot
self.imsclient.deploy_instrument_device(newInstDevice_id, newDeployment_id)
self.omsclient.deploy_instrument_site(instrumentSite_id, newDeployment_id)
# -------------------------------
# Create InstrumentAgentInstance for NewInstrumentDevice to hold configuration information
# -------------------------------
port_agent_config = {
"device_addr": "sbe37-simulator.oceanobservatories.org",
"device_port": 4004,
"process_type": PortAgentProcessType.UNIX,
"binary_path": "port_agent",
"port_agent_addr": "localhost",
"command_port": 4005,
"data_port": 4006,
"log_level": 5,
"type": PortAgentType.ETHERNET,
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstanceYear2",
description="SBE37IMAgentInstanceYear2",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4004,
port_agent_config=port_agent_config,
)
newInstAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, newInstDevice_id
)
# -------------------------------
# Create CTD Parsed as the Year 2 data product
# -------------------------------
dp_obj = IonObject(
RT.DataProduct,
name="ctd_parsed_year2",
description="ctd stream test year 2",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_parsed_data_product_year2 = self.dataproductclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id
)
print "new ctd_parsed_data_product_id = ", ctd_parsed_data_product_year2
self.damsclient.assign_data_product(
input_resource_id=newInstDevice_id, data_product_id=ctd_parsed_data_product_year2
)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_year2, PRED.hasStream, None, True)
print "test_deployAsPrimaryDevice: Data product streams2 = ", stream_ids
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
# -------------------------------
log.debug("test_deployAsPrimaryDevice: create data process definition ctd_L0_all")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L0_all",
description="transform ctd package into three separate L0 streams",
module="ion.processes.data.transforms.ctd.ctd_L0_all",
class_name="ctd_L0_all",
)
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" % ex)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
# -------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name="L0_Conductivity", parameter_dictionary_id=parsed_pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding="conductivity"
)
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name="L0_Pressure", parameter_dictionary_id=parsed_pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding="pressure"
)
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name="L0_Temperature", parameter_dictionary_id=parsed_pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding="temperature"
)
self.output_products = {}
log.debug("test_deployAsPrimaryDevice: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Conductivity",
description="transform output conductivity",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_conductivity_output_dp_obj, stream_definition_id=parsed_stream_def_id
)
self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id
# self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_deployAsPrimaryDevice: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Pressure",
description="transform output pressure",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_pressure_output_dp_obj, stream_definition_id=parsed_stream_def_id
)
self.output_products["pressure"] = ctd_l0_pressure_output_dp_id
# self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_deployAsPrimaryDevice: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Temperature",
description="transform output temperature",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
data_product=ctd_l0_temperature_output_dp_obj, stream_definition_id=parsed_stream_def_id
)
self.output_products["temperature"] = ctd_l0_temperature_output_dp_id
# self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process, listening to Sim1 (later: logical instrument output product)
# -------------------------------
log.debug("test_deployAsPrimaryDevice: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [ctd_parsed_data_product_year1], self.output_products
)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug("test_deployAsPrimaryDevice: create L0 all data_process return")
# --------------------------------
# Activate the deployment
# --------------------------------
self.omsclient.activate_deployment(oldDeployment_id)
# -------------------------------
# Launch InstrumentAgentInstance Sim1, connect to the resource agent client
# -------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=oldInstAgentInstance_id)
self.addCleanup(
self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=oldInstAgentInstance_id
)
# wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id)
gate = ProcessStateGate(
self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING
)
self.assertTrue(
gate.await(30),
"The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id,
)
inst_agent1_instance_obj = self.imsclient.read_instrument_agent_instance(oldInstAgentInstance_id)
print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent1_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim1 = ResourceAgentClient(
"iaclient Sim1", name=inst_agent1_instance_obj.agent_process_id, process=FakeProcess()
)
print "activate_instrument: got _ia_client_sim1 %s", self._ia_client_sim1
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim1 %s", str(self._ia_client_sim1))
# -------------------------------
# Launch InstrumentAgentInstance Sim2, connect to the resource agent client
# -------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=newInstAgentInstance_id)
self.addCleanup(
self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=newInstAgentInstance_id
)
# wait for start
instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id)
gate = ProcessStateGate(
self.processdispatchclient.read_process, instance_obj.agent_process_id, ProcessStateEnum.RUNNING
)
self.assertTrue(
gate.await(30),
"The instrument agent instance (%s) did not spawn in 30 seconds" % instance_obj.agent_process_id,
)
inst_agent2_instance_obj = self.imsclient.read_instrument_agent_instance(newInstAgentInstance_id)
print "test_deployAsPrimaryDevice: Instrument agent instance obj: = ", inst_agent2_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client_sim2 = ResourceAgentClient(
"iaclient Sim2", name=inst_agent2_instance_obj.agent_process_id, process=FakeProcess()
)
print "activate_instrument: got _ia_client_sim2 %s", self._ia_client_sim2
log.debug(" test_deployAsPrimaryDevice:: got _ia_client_sim2 %s", str(self._ia_client_sim2))
# -------------------------------
# Streaming Sim1 (old instrument)
# -------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: initialize %s", str(retval))
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: return value from go_active %s", str(reply))
self.assertTrue(reply)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client_sim1.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: run %s", str(reply))
gevent.sleep(2)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client_sim1.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE: %s", str(retval))
# -------------------------------
# Streaming Sim 2 (new instrument)
# -------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: initialize_sim2 %s", str(retval))
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: return value from go_active_sim2 %s", str(reply))
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client_sim2.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active_sim2 command %s", str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client_sim2.execute_agent(cmd)
log.debug("test_deployAsPrimaryDevice: run %s", str(reply))
gevent.sleep(2)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client_sim2.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from START_AUTOSAMPLE_sim2: %s", str(retval))
gevent.sleep(10)
# -------------------------------
# Shutdown Sim1 (old instrument)
# -------------------------------
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client_sim1.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE: %s", str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s", str(reply))
time.sleep(5)
# -------------------------------
# Shutdown Sim2 (old instrument)
# -------------------------------
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client_sim2.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from STOP_AUTOSAMPLE_sim2: %s", str(retval))
log.debug("test_activateInstrumentSample: calling reset_sim2 ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client_sim1.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset_sim2 %s", str(reply))
time.sleep(5)
class TestCTDTransformsIntegration(IonIntegrationTestCase):
pdict_id = None
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataset_management = self.datasetclient
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_instrument_model(self):
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
return instModel_id
def _create_instrument_agent(self, instModel_id):
raw_config = StreamConfiguration(
stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
return instAgent_id
def _create_instrument_device(self, instModel_id):
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
return instDevice_id
def _create_instrument_agent_instance(self, instAgent_id, instDevice_id):
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
return instAgentInstance_id
def _create_param_dicts(self):
tdom, sdom = time_series_domain()
self.sdom = sdom.dump()
self.tdom = tdom.dump()
self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
def _create_input_data_products(
self,
ctd_stream_def_id,
instDevice_id,
):
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(
dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_parsed_data_product)
#---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
#---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product,
PRED.hasStream, None, True)
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
#---------------------------------------------------------------------------
# Create CTD Raw as the second data product
#---------------------------------------------------------------------------
if not self.pdict_id:
self._create_param_dicts()
raw_stream_def_id = self.pubsubclient.create_stream_definition(
name='SBE37_RAW', parameter_dictionary_id=self.pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(
dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_raw_data_product)
#---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
#---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product,
PRED.hasStream, None, True)
#---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
#---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product,
PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
return ctd_parsed_data_product
def _create_data_process_definitions(self):
#-------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L2 Density: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
return self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id,\
self.ctd_L1_pressure_dprocdef_id,self.ctd_L1_temperature_dprocdef_id, \
self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id
def _create_stream_definitions(self):
if not self.pdict_id:
self._create_param_dicts()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name='L0_Conductivity', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id,
self.ctd_L0_all_dprocdef_id,
binding='conductivity')
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name='L0_Pressure', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id,
self.ctd_L0_all_dprocdef_id,
binding='pressure')
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name='L0_Temperature', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id,
self.ctd_L0_all_dprocdef_id,
binding='temperature')
return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id
def _create_l0_output_data_products(self,
outgoing_stream_l0_conductivity_id,
outgoing_stream_l0_pressure_id,
outgoing_stream_l0_temperature_id):
out_data_prods = []
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
out_data_prods.append(self.ctd_l0_conductivity_output_dp_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l0_conductivity_output_dp_id)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
out_data_prods.append(self.ctd_l0_pressure_output_dp_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l0_pressure_output_dp_id)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
out_data_prods.append(self.ctd_l0_temperature_output_dp_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l0_temperature_output_dp_id)
return out_data_prods
def _create_l1_out_data_products(self):
ctd_l1_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_conductivity_output_dp_obj,
self.outgoing_stream_l1_conductivity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l1_conductivity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_conductivity', stream_ids[0])
self.loggerpids.append(pid)
ctd_l1_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l1_pressure_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_pressure', stream_ids[0])
self.loggerpids.append(pid)
ctd_l1_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_temperature_output_dp_obj,
self.outgoing_stream_l1_temperature_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l1_temperature_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_temperature', stream_ids[0])
self.loggerpids.append(pid)
def _create_l2_out_data_products(self):
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
if not self.pdict_id: self._create_param_dicts()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(
name='L2_salinity', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_salinity_id,
self.ctd_L2_salinity_dprocdef_id,
binding='salinity')
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(
name='L2_Density', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_density_id,
self.ctd_L2_density_dprocdef_id,
binding='density')
ctd_l2_salinity_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l2_salinity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_salinity', stream_ids[0])
self.loggerpids.append(pid)
ctd_l2_density_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Density',
description='transform output pressure',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l2_density_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_density', stream_ids[0])
self.loggerpids.append(pid)
@unittest.skip('This test errors on coi-nightly, may be OBE.')
def test_createTransformsThenActivateInstrument(self):
self.loggerpids = []
#-------------------------------------------------------------------------------------
# Create InstrumentModel
#-------------------------------------------------------------------------------------
instModel_id = self._create_instrument_model()
#-------------------------------------------------------------------------------------
# Create InstrumentAgent
#-------------------------------------------------------------------------------------
instAgent_id = self._create_instrument_agent(instModel_id)
#-------------------------------------------------------------------------------------
# Create InstrumentDevice
#-------------------------------------------------------------------------------------
instDevice_id = self._create_instrument_device(instModel_id)
#-------------------------------------------------------------------------------------
# Create Instrument Agent Instance
#-------------------------------------------------------------------------------------
instAgentInstance_id = self._create_instrument_agent_instance(
instAgent_id, instDevice_id)
#-------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#-------------------------------------------------------------------------------------
self._create_param_dicts()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
name='SBE37_CDM', parameter_dictionary_id=self.pdict_id)
#-------------------------------------------------------------------------------------
# Create two data products
#-------------------------------------------------------------------------------------
ctd_parsed_data_product = self._create_input_data_products(
ctd_stream_def_id, instDevice_id)
#-------------------------------------------------------------------------------------
# Create data process definitions
#-------------------------------------------------------------------------------------
self._create_data_process_definitions()
#-------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------------------------------------------------------------
outgoing_stream_l0_conductivity_id, \
outgoing_stream_l0_pressure_id, \
outgoing_stream_l0_temperature_id = self._create_stream_definitions()
self.out_prod_ids = self._create_l0_output_data_products(
outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id,
outgoing_stream_l0_temperature_id)
self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(
name='L1_conductivity', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_conductivity_id,
self.ctd_L1_conductivity_dprocdef_id,
binding='conductivity')
self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(
name='L1_Pressure', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_pressure_id,
self.ctd_L1_pressure_dprocdef_id,
binding='pressure')
self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(
name='L1_Temperature', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_temperature_id,
self.ctd_L1_temperature_dprocdef_id,
binding='temperature')
self._create_l1_out_data_products()
self._create_l2_out_data_products()
#-------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------------------------------------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=self.ctd_L0_all_dprocdef_id,
in_data_product_ids=[ctd_parsed_data_product],
out_data_product_ids=self.out_prod_ids)
self.dataprocessclient.activate_data_process(
ctd_l0_all_data_process_id)
data_process = self.rrclient.read(ctd_l0_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=ctd_l0_all_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
extended_process = self.dataprocessclient.get_data_process_extension(
ctd_l0_all_data_process_id)
self.assertEquals(extended_process.computed.operational_state.status,
ComputedValueAvailability.NOTAVAILABLE)
self.assertEquals(data_process.message_controllable, True)
#-------------------------------------------------------------------------------------
# L1 Conductivity: Create the data process
#-------------------------------------------------------------------------------------
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_conductivity_dprocdef_id,
[self.ctd_l0_conductivity_output_dp_id],
[self.ctd_l1_conductivity_output_dp_id])
self.dataprocessclient.activate_data_process(
l1_conductivity_data_process_id)
data_process = self.rrclient.read(l1_conductivity_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l1_conductivity_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# L1 Pressure: Create the data process
#-------------------------------------------------------------------------------------
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_pressure_dprocdef_id,
[self.ctd_l0_pressure_output_dp_id],
[self.ctd_l1_pressure_output_dp_id])
self.dataprocessclient.activate_data_process(
l1_pressure_data_process_id)
data_process = self.rrclient.read(l1_pressure_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l1_pressure_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# L1 Temperature: Create the data process
#-------------------------------------------------------------------------------------
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_temperature_dprocdef_id,
[self.ctd_l0_temperature_output_dp_id],
[self.ctd_l1_temperature_output_dp_id])
self.dataprocessclient.activate_data_process(
l1_temperature_all_data_process_id)
data_process = self.rrclient.read(l1_temperature_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l1_temperature_all_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# L2 Salinity: Create the data process
#-------------------------------------------------------------------------------------
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product],
[self.ctd_l2_salinity_output_dp_id])
self.dataprocessclient.activate_data_process(
l2_salinity_all_data_process_id)
data_process = self.rrclient.read(l2_salinity_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l2_salinity_all_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# L2 Density: Create the data process
#-------------------------------------------------------------------------------------
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product],
[self.ctd_l2_density_output_dp_id])
self.dataprocessclient.activate_data_process(
l2_density_all_data_process_id)
data_process = self.rrclient.read(l2_density_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l2_density_all_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------------------------------------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
# Wait for instrument agent to spawn
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id, ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (15),
"The instrument agent instance did not spawn in 15 seconds")
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
#-------------------------------------------------------------------------------------
# Streaming
#-------------------------------------------------------------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
#todo ResourceAgentClient no longer has method set_param
# # Make sure the sampling rate and transmission are sane.
# params = {
# SBE37Parameter.NAVG : 1,
# SBE37Parameter.INTERVAL : 5,
# SBE37Parameter.TXREALTIME : True
# }
# self._ia_client.set_param(params)
#todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it?
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# This gevent sleep is there to test the autosample time, which will show something different from default
# only if the instrument runs for over a minute
gevent.sleep(90)
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
autosample_string = extended_instrument.computed.uptime.value
autosampling_time = int(autosample_string.split()[4])
self.assertTrue(autosampling_time > 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
#todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it?
# log.debug("test_activateInstrumentStream: calling go_observatory")
# cmd = AgentCommand(command='go_observatory')
# reply = self._ia_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
#-------------------------------------------------------------------------------------------------
# Cleanup processes
#-------------------------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
#--------------------------------------------------------------------------------
# Cleanup data products
#--------------------------------------------------------------------------------
dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct,
id_only=True)
for dp_id in dp_ids:
self.dataproductclient.delete_data_product(dp_id)
class TestCTDPChain(IonIntegrationTestCase):
def setUp(self):
super(TestCTDPChain, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.dataproduct_management = DataProductManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
# This is for the time values inside the packets going into the transform
self.i = 0
self.cnt = 0
# Cleanup of queue created by the subscriber
self.queue_cleanup = []
self.data_process_cleanup = []
def _get_new_ctd_L0_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
rdt['time'] = numpy.arange(self.i, self.i+length)
for field in rdt:
if isinstance(rdt._pdict.get_context(field).param_type, QuantityType):
rdt[field] = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)])
g = rdt.to_granule()
self.i+=length
return g
def clean_queues(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
def cleaning_operations(self):
for dproc_id in self.data_process_cleanup:
self.data_process_management.delete_data_process(dproc_id)
def test_ctdp_chain(self):
"""
Test that packets are processed by a chain of CTDP transforms: L0, L1 and L2
"""
#-------------------------------------------------------------------------------------
# Prepare the stream def to be used for transform chain
#-------------------------------------------------------------------------------------
#todo Check whether the right parameter dictionary is being used
self._prepare_stream_def_for_transform_chain()
#-------------------------------------------------------------------------------------
# Prepare the data proc defs and in and out data products for the transforms
#-------------------------------------------------------------------------------------
# list_args_L0 = [data_proc_def_id, input_dpod_id, output_dpod_id]
list_args_L0 = self._prepare_things_you_need_to_launch_transform(name_of_transform='L0')
list_args_L1 = self._prepare_things_you_need_to_launch_transform(name_of_transform='L1')
list_args_L2_density = self._prepare_things_you_need_to_launch_transform(name_of_transform='L2_density')
list_args_L2_salinity = self._prepare_things_you_need_to_launch_transform(name_of_transform='L2_salinity')
log.debug("Got the following args: L0 = %s, L1 = %s, L2 density = %s, L2 salinity = %s", list_args_L0, list_args_L1, list_args_L2_density, list_args_L2_salinity )
#-------------------------------------------------------------------------------------
# Launch the CTDP transforms
#-------------------------------------------------------------------------------------
L0_data_proc_id = self._launch_transform('L0', *list_args_L0)
L1_data_proc_id = self._launch_transform('L1', *list_args_L1)
L2_density_data_proc_id = self._launch_transform('L2_density', *list_args_L2_density)
L2_salinity_data_proc_id = self._launch_transform('L2_salinity', *list_args_L2_salinity)
log.debug("Launched the transforms: L0 = %s, L1 = %s", L0_data_proc_id, L1_data_proc_id)
#-------------------------------------------------------------------------
# Start a subscriber listening to the output of each of the transforms
#-------------------------------------------------------------------------
ar_L0 = self.start_subscriber_listening_to_L0_transform(out_data_prod_id=list_args_L0[2])
ar_L1 = self.start_subscriber_listening_to_L1_transform(out_data_prod_id=list_args_L1[2])
ar_L2_density = self.start_subscriber_listening_to_L2_density_transform(out_data_prod_id=list_args_L2_density[2])
ar_L2_salinity = self.start_subscriber_listening_to_L2_density_transform(out_data_prod_id=list_args_L2_salinity[2])
#-------------------------------------------------------------------
# Publish the parsed packets that the L0 transform is listening for
#-------------------------------------------------------------------
stream_id, stream_route = self.get_stream_and_route_for_data_prod(data_prod_id= list_args_L0[1])
self._publish_for_L0_transform(stream_id, stream_route)
#-------------------------------------------------------------------
# Check the granules being outputted by the transforms
#-------------------------------------------------------------------
self._check_granule_from_L0_transform(ar_L0)
self._check_granule_from_L1_transform(ar_L1)
self._check_granule_from_L2_density_transform(ar_L2_density)
self._check_granule_from_L2_salinity_transform(ar_L2_salinity)
def _prepare_stream_def_for_transform_chain(self):
# Get the stream definition for the stream using the parameter dictionary
# pdict_id = self.dataset_management.read_parameter_dictionary_by_name(parameter_dict_name, id_only=True)
pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'input_param_for_L0')
self.in_stream_def_id_for_L0 = self.pubsub.create_stream_definition(name='stream_def_for_L0', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub.delete_stream_definition, self.in_stream_def_id_for_L0)
pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'params_for_other_transforms')
self.stream_def_id = self.pubsub.create_stream_definition(name='stream_def_for_CTDBP_transforms', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub.delete_stream_definition, self.stream_def_id)
log.debug("Got the parsed parameter dictionary: id: %s", pdict_id)
log.debug("Got the stream def for parsed input to L0: %s", self.in_stream_def_id_for_L0)
log.debug("Got the stream def for other other streams: %s", self.stream_def_id)
def _prepare_things_you_need_to_launch_transform(self, name_of_transform = ''):
module, class_name = self._get_class_module(name_of_transform)
#-------------------------------------------------------------------------
# Data Process Definition
#-------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name= 'CTDBP_%s_Transform' % name_of_transform,
description= 'Data Process Definition for the CTDBP %s transform.' % name_of_transform,
module= module,
class_name=class_name)
data_proc_def_id = self.data_process_management.create_data_process_definition(dpd_obj)
self.addCleanup(self.data_process_management.delete_data_process_definition, data_proc_def_id)
log.debug("created data process definition: id = %s", data_proc_def_id)
#-------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects for the data product objects
#-------------------------------------------------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
#-------------------------------------------------------------------------
# Get the names of the input and output data products
#-------------------------------------------------------------------------
input_dpod_id = ''
output_dpod_id = ''
if name_of_transform == 'L0':
input_dpod_id = self._create_input_data_product('parsed', tdom, sdom)
output_dpod_id = self._create_output_data_product('L0', tdom, sdom)
self.in_prod_for_L1 = output_dpod_id
elif name_of_transform == 'L1':
input_dpod_id = self.in_prod_for_L1
output_dpod_id = self._create_output_data_product('L1', tdom, sdom)
self.in_prod_for_L2 = output_dpod_id
elif name_of_transform == 'L2_density':
input_dpod_id = self.in_prod_for_L2
output_dpod_id = self._create_output_data_product('L2_density', tdom, sdom)
elif name_of_transform == 'L2_salinity':
input_dpod_id = self.in_prod_for_L2
output_dpod_id = self._create_output_data_product('L2_salinity', tdom, sdom)
else:
self.fail("something bad happened")
return [data_proc_def_id, input_dpod_id, output_dpod_id]
def _get_class_module(self, name_of_transform):
options = {'L0' : self._class_module_L0,
'L1' : self._class_module_L1,
'L2_density' : self._class_module_L2_density,
'L2_salinity' : self._class_module_L2_salinity}
return options[name_of_transform]()
def _class_module_L0(self):
module = 'ion.processes.data.transforms.ctdbp.ctdbp_L0'
class_name = 'CTDBP_L0_all'
return module, class_name
def _class_module_L1(self):
module = 'ion.processes.data.transforms.ctdbp.ctdbp_L1'
class_name = 'CTDBP_L1_Transform'
return module, class_name
def _class_module_L2_density(self):
module = 'ion.processes.data.transforms.ctdbp.ctdbp_L2_density'
class_name = 'CTDBP_DensityTransform'
return module, class_name
def _class_module_L2_salinity(self):
module = 'ion.processes.data.transforms.ctdbp.ctdbp_L2_salinity'
class_name = 'CTDBP_SalinityTransform'
return module, class_name
def _create_input_data_product(self, name_of_transform = '', tdom = None, sdom = None):
dpod_obj = IonObject(RT.DataProduct,
name='dprod_%s' % name_of_transform,
description='for_%s' % name_of_transform,
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("the stream def id: %s", self.stream_def_id)
if name_of_transform == 'L0':
stream_def_id = self.in_stream_def_id_for_L0
else:
stream_def_id = self.stream_def_id
dpod_id = self.dataproduct_management.create_data_product(data_product=dpod_obj,
stream_definition_id= stream_def_id
)
self.addCleanup(self.dataproduct_management.delete_data_product, dpod_id)
log.debug("got the data product out. id: %s", dpod_id)
return dpod_id
def _create_output_data_product(self, name_of_transform = '', tdom = None, sdom = None):
dpod_obj = IonObject(RT.DataProduct,
name='dprod_%s' % name_of_transform,
description='for_%s' % name_of_transform,
temporal_domain = tdom,
spatial_domain = sdom)
if name_of_transform == 'L0':
stream_def_id = self.in_stream_def_id_for_L0
else:
stream_def_id = self.stream_def_id
dpod_id = self.dataproduct_management.create_data_product(data_product=dpod_obj,
stream_definition_id=stream_def_id
)
self.addCleanup(self.dataproduct_management.delete_data_product, dpod_id)
return dpod_id
def _launch_transform(self, name_of_transform = '', data_proc_def_id = None, input_dpod_id = None, output_dpod_id = None):
# We need the key name here to be "L2_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L2_stream when the config is used to launch the data process
if name_of_transform in ['L0', 'L1']:
binding = '%s_stream' % name_of_transform
elif name_of_transform == 'L2_salinity':
binding = 'salinity'
elif name_of_transform == 'L2_density':
binding = 'density'
config = None
if name_of_transform == 'L1':
config = self._create_calibration_coefficients_dict()
elif name_of_transform == 'L2_density':
config = DotDict()
config.process = {'lat' : 32.7153, 'lon' : 117.1564}
log.debug("launching transform for name: %s",name_of_transform )
log.debug("launching transform for data_proc_def_id: %s\ninput_dpod_id: %s\noutput_dpod_id: %s", data_proc_def_id, input_dpod_id, output_dpod_id )
data_proc_id = self.data_process_management.create_data_process(
data_process_definition_id = data_proc_def_id,
in_data_product_ids= [input_dpod_id],
out_data_product_ids = [output_dpod_id],
configuration = config)
self.addCleanup(self.data_process_management.delete_data_process, data_proc_id)
self.data_process_management.activate_data_process(data_proc_id)
self.addCleanup(self.data_process_management.deactivate_data_process, data_proc_id)
log.debug("Created a data process for ctdbp %s transform: id = %s", name_of_transform, data_proc_id)
return data_proc_id
def get_stream_and_route_for_data_prod(self, data_prod_id = ''):
stream_ids, _ = self.resource_registry.find_objects(data_prod_id, PRED.hasStream, RT.Stream, True)
stream_id = stream_ids[0]
input_stream = self.resource_registry.read(stream_id)
stream_route = input_stream.stream_route
return stream_id, stream_route
def start_subscriber_listening_to_L0_transform(self, out_data_prod_id = ''):
#----------- Create subscribers to listen to the two transforms --------------------------------
stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True)
output_stream_id_of_transform = stream_ids[0]
ar_L0 = self._start_subscriber_to_transform( name_of_transform = 'L0',stream_id=output_stream_id_of_transform)
return ar_L0
def start_subscriber_listening_to_L1_transform(self, out_data_prod_id = ''):
#----------- Create subscribers to listen to the two transforms --------------------------------
stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True)
output_stream_id_of_transform = stream_ids[0]
ar_L1 = self._start_subscriber_to_transform( name_of_transform = 'L1',stream_id=output_stream_id_of_transform)
return ar_L1
def start_subscriber_listening_to_L2_density_transform(self, out_data_prod_id = ''):
#----------- Create subscribers to listen to the two transforms --------------------------------
stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True)
output_stream_id_of_transform = stream_ids[0]
ar_L2_density = self._start_subscriber_to_transform( name_of_transform = 'L2_density', stream_id=output_stream_id_of_transform)
return ar_L2_density
def start_subscriber_listening_to_L2_salinity_transform(self, out_data_prod_id = ''):
#----------- Create subscribers to listen to the two transforms --------------------------------
stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True)
output_stream_id_of_transform = stream_ids[0]
ar_L2_density = self._start_subscriber_to_transform( name_of_transform = 'L2_salinity',stream_id=output_stream_id_of_transform)
return ar_L2_density
def _start_subscriber_to_transform(self, name_of_transform = '', stream_id = ''):
ar = gevent.event.AsyncResult()
def subscriber(m,r,s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub_%s' % name_of_transform, callback=subscriber)
# Note that this running the below line creates an exchange since none of that name exists before
sub_id = self.pubsub.create_subscription('subscriber_to_transform_%s' % name_of_transform,
stream_ids=[stream_id],
exchange_name='sub_%s' % name_of_transform)
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
return ar
def _check_granule_from_L0_transform(self, ar = None):
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the L0 transform: %s", granule_from_transform)
# Check the algorithm being applied
self._check_application_of_L0_algorithm(granule_from_transform)
def _check_granule_from_L1_transform(self, ar = None):
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the L1 transform: %s", granule_from_transform)
# Check the algorithm being applied
self._check_application_of_L1_algorithm(granule_from_transform)
def _check_granule_from_L2_density_transform(self, ar = None):
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the L2 transform: %s", granule_from_transform)
# Check the algorithm being applied
self._check_application_of_L2_density_algorithm(granule_from_transform)
def _check_granule_from_L2_salinity_transform(self, ar = None):
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the L2 transform: %s", granule_from_transform)
# Check the algorithm being applied
self._check_application_of_L2_salinity_algorithm(granule_from_transform)
def _check_application_of_L0_algorithm(self, granule = None):
""" Check the algorithm applied by the L0 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = ['time', 'pressure', 'conductivity', 'temperature']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def _check_application_of_L1_algorithm(self, granule = None):
""" Check the algorithm applied by the L1 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = [ 'time', 'pressure', 'conductivity', 'temp']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def _check_application_of_L2_density_algorithm(self, granule = None):
""" Check the algorithm applied by the L2 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = ['time', 'density']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def _check_application_of_L2_salinity_algorithm(self, granule = None):
""" Check the algorithm applied by the L2 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = ['time', 'salinity']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def _publish_for_L0_transform(self, input_stream_id = None, stream_route = None):
#----------- Publish on that stream so that the transform can receive it --------------------------------
self._publish_to_transform(input_stream_id, stream_route )
def _publish_to_transform(self, stream_id = '', stream_route = None):
pub = StandaloneStreamPublisher(stream_id, stream_route)
publish_granule = self._get_new_ctd_L0_packet(stream_definition_id=self.in_stream_def_id_for_L0, length = 5)
pub.publish(publish_granule)
log.debug("Published the following granule: %s", publish_granule)
def _create_input_param_dict_for_test(self, parameter_dict_name = ''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'Siemens_per_meter'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'Pascal'
pdict.add_context(pres_ctxt)
if parameter_dict_name == 'input_param_for_L0':
temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
else:
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_kelvin'
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = 'g/m'
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = 'PSU'
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == 'input_param_for_L0':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
elif pc[1].name == 'temp':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
def _create_calibration_coefficients_dict(self):
config = DotDict()
config.process.calibration_coeffs = {
'temp_calibration_coeffs': {
'TA0' : 1.561342e-03,
'TA1' : 2.561486e-04,
'TA2' : 1.896537e-07,
'TA3' : 1.301189e-07,
'TOFFSET' : 0.000000e+00
},
'cond_calibration_coeffs': {
'G' : -9.896568e-01,
'H' : 1.316599e-01,
'I' : -2.213854e-04,
'J' : 3.292199e-05,
'CPCOR' : -9.570000e-08,
'CTCOR' : 3.250000e-06,
'CSLOPE' : 1.000000e+00
},
'pres_calibration_coeffs' : {
'PA0' : 4.960417e-02,
'PA1' : 4.883682e-04,
'PA2' : -5.687309e-12,
'PTCA0' : 5.249802e+05,
'PTCA1' : 7.595719e+00,
'PTCA2' : -1.322776e-01,
'PTCB0' : 2.503125e+01,
'PTCB1' : 5.000000e-05,
'PTCB2' : 0.000000e+00,
'PTEMPA0' : -6.431504e+01,
'PTEMPA1' : 5.168177e+01,
'PTEMPA2' : -2.847757e-01,
'POFFSET' : 0.000000e+00
}
}
return config
class TestOmsLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.platformModel_id = None
# rsn_oms: to retrieve network structure and information from RSN-OMS:
# Note that OmsClientFactory will create an "embedded" RSN OMS
# simulator object by default.
self.rsn_oms = OmsClientFactory.create_instance()
self.all_platforms = {}
self.topology = {}
self.agent_device_map = {}
self.agent_streamconfig_map = {}
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
self._set_up_DataProduct_obj()
self._set_up_PlatformModel_obj()
def _set_up_DataProduct_obj(self):
# Create data product object to be used for each of the platform log streams
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name('platform_eng_parsed', id_only=True)
self.platform_eng_stream_def_id = self.pubsubcli.create_stream_definition(
name='platform_eng', parameter_dictionary_id=self.pdict_id)
self.dp_obj = IonObject(RT.DataProduct,
name='platform_eng data',
description='platform_eng test',
temporal_domain = tdom,
spatial_domain = sdom)
def _set_up_PlatformModel_obj(self):
# Create PlatformModel
platformModel_obj = IonObject(RT.PlatformModel,
name='RSNPlatformModel',
description="RSNPlatformModel")
try:
self.platformModel_id = self.imsclient.create_platform_model(platformModel_obj)
except BadRequest as ex:
self.fail("failed to create new PLatformModel: %s" %ex)
log.debug( 'new PlatformModel id = %s', self.platformModel_id)
def _traverse(self, platform_id, parent_platform_objs=None):
"""
Recursive routine that repeatedly calls _prepare_platform to build
the object dictionary for each platform.
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval the dict returned by _prepare_platform at this level.
"""
log.info("Starting _traverse for %r", platform_id)
plat_objs = self._prepare_platform(platform_id, parent_platform_objs)
self.all_platforms[platform_id] = plat_objs
# now, traverse the children:
retval = self.rsn_oms.getSubplatformIDs(platform_id)
subplatform_ids = retval[platform_id]
for subplatform_id in subplatform_ids:
self._traverse(subplatform_id, plat_objs)
# note, topology indexed by platform_id
self.topology[platform_id] = plat_objs['children']
return plat_objs
def _prepare_platform(self, platform_id, parent_platform_objs):
"""
This routine generalizes the manual construction currently done in
test_oms_launch.py. It is called by the recursive _traverse method so
all platforms starting from a given base platform are prepared.
Note: For simplicity in this test, sites are organized in the same
hierarchical way as the platforms themselves.
@param platform_id ID of the platform to be visited
@param parent_platform_objs dict of objects associated to parent
platform, if any.
@retval a dict of associated objects similar to those in
test_oms_launch
"""
site__obj = IonObject(RT.PlatformSite,
name='%s_PlatformSite' % platform_id,
description='%s_PlatformSite platform site' % platform_id)
site_id = self.omsclient.create_platform_site(site__obj)
if parent_platform_objs:
# establish hasSite association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['site_id'],
predicate=PRED.hasSite,
object=site_id)
# prepare platform attributes and ports:
monitor_attribute_objs, monitor_attribute_dicts = self._prepare_platform_attributes(platform_id)
port_objs, port_dicts = self._prepare_platform_ports(platform_id)
device__obj = IonObject(RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
# ports=port_objs,
# platform_monitor_attributes = monitor_attribute_objs
)
device__dict = dict(ports=port_dicts,
platform_monitor_attributes=monitor_attribute_dicts)
self.device_id = self.imsclient.create_platform_device(device__obj)
self.imsclient.assign_platform_model_to_platform_device(self.platformModel_id, self.device_id)
self.rrclient.create_association(subject=site_id, predicate=PRED.hasDevice, object=self.device_id)
self.damsclient.register_instrument(instrument_id=self.device_id)
if parent_platform_objs:
# establish hasDevice association with the parent
self.rrclient.create_association(
subject=parent_platform_objs['device_id'],
predicate=PRED.hasDevice,
object=self.device_id)
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' % platform_id)
agent_id = self.imsclient.create_platform_agent(agent__obj)
if parent_platform_objs:
# add this platform_id to parent's children:
parent_platform_objs['children'].append(platform_id)
self.imsclient.assign_platform_model_to_platform_agent(self.platformModel_id, agent_id)
# agent_instance_obj = IonObject(RT.PlatformAgentInstance,
# name='%s_PlatformAgentInstance' % platform_id,
# description="%s_PlatformAgentInstance" % platform_id)
#
# agent_instance_id = self.imsclient.create_platform_agent_instance(
# agent_instance_obj, agent_id, device_id)
plat_objs = {
'platform_id': platform_id,
'site__obj': site__obj,
'site_id': site_id,
'device__obj': device__obj,
'device_id': self.device_id,
'agent__obj': agent__obj,
'agent_id': agent_id,
# 'agent_instance_obj': agent_instance_obj,
# 'agent_instance_id': agent_instance_id,
'children': []
}
log.info("plat_objs for platform_id %r = %s", platform_id, str(plat_objs))
self.agent_device_map[platform_id] = device__dict
# self.agent_device_map[platform_id] = device__obj
stream_config = self._create_stream_config(plat_objs)
self.agent_streamconfig_map[platform_id] = stream_config
# self.agent_streamconfig_map[platform_id] = None
# self._start_data_subscriber(agent_instance_id, stream_config)
return plat_objs
def _prepare_platform_attributes(self, platform_id):
"""
Returns the list of PlatformMonitorAttributes objects corresponding to
the attributes associated to the given platform.
"""
result = self.rsn_oms.getPlatformAttributes(platform_id)
self.assertTrue(platform_id in result)
ret_infos = result[platform_id]
monitor_attribute_objs = []
monitor_attribute_dicts = []
for attrName, attrDfn in ret_infos.iteritems():
log.debug("platform_id=%r: preparing attribute=%r", platform_id, attrName)
monitor_rate = attrDfn['monitorCycleSeconds']
units = attrDfn['units']
plat_attr_obj = IonObject(OT.PlatformMonitorAttributes,
id=attrName,
monitor_rate=monitor_rate,
units=units)
plat_attr_dict = dict(id=attrName,
monitor_rate=monitor_rate,
units=units)
monitor_attribute_objs.append(plat_attr_obj)
monitor_attribute_dicts.append(plat_attr_dict)
return monitor_attribute_objs, monitor_attribute_dicts
def _prepare_platform_ports(self, platform_id):
"""
Returns the list of PlatformPort objects corresponding to the ports
associated to the given platform.
"""
result = self.rsn_oms.getPlatformPorts(platform_id)
self.assertTrue(platform_id in result)
port_dict = result[platform_id]
port_objs = []
port_dicts = []
for port_id, port in port_dict.iteritems():
log.debug("platform_id=%r: preparing port=%r", platform_id, port_id)
ip_address = port['comms']['ip']
plat_port_obj = IonObject(OT.PlatformPort,
port_id=port_id,
ip_address=ip_address)
plat_port_dict = dict(port_id=port_id,
ip_address=ip_address)
port_objs.append(plat_port_obj)
port_dicts.append(plat_port_dict)
return port_objs, port_dicts
def _create_stream_config(self, plat_objs):
platform_id = plat_objs['platform_id']
device_id = plat_objs['device_id']
#create the log data product
self.dp_obj.name = '%s platform_eng data' % platform_id
self.data_product_id = self.dpclient.create_data_product(data_product=self.dp_obj, stream_definition_id=self.platform_eng_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=self.device_id, data_product_id=self.data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(self.data_product_id, PRED.hasStream, None, True)
stream_config = self._build_stream_config(stream_ids[0])
return stream_config
def _build_stream_config(self, stream_id=''):
platform_eng_dictionary = DatasetManagementService.get_parameter_dictionary_by_name('platform_eng_parsed')
#get the streamroute object from pubsub by passing the stream_id
stream_def_ids, _ = self.rrclient.find_objects(stream_id,
PRED.hasStreamDefinition,
RT.StreamDefinition,
True)
stream_route = self.pubsubcli.read_stream_route(stream_id=stream_id)
stream_config = {'routing_key' : stream_route.routing_key,
'stream_id' : stream_id,
'stream_definition_ref' : stream_def_ids[0],
'exchange_point' : stream_route.exchange_point,
'parameter_dictionary':platform_eng_dictionary.dump()}
return stream_config
def _set_platform_agent_instances(self):
"""
Once most of the objs/defs associated with all platforms are in
place, this method creates and associates the PlatformAgentInstance
elements.
"""
admap = self.agent_device_map
#############################################################
# Note, if using PlatformDevice objects in self.agent_device_map, the
# following is an attempt to serialize it and then pass it in the
# platform config dict below. However, although the output seems
# properly serialized), the following error is generated:
# BadRequest: 400 - bad configuration: <interface.objects.PlatformDevice object at 0x115504b50> is not JSON serializable
# So, I'm commenting out this attempt for the moment.
# Instead self.agent_device_map is created in terms of dictionaries.
# from pyon.core.object import ion_serializer
# admap = ion_serializer.serialize(admap)
#############################################################
log.info("agent_device_map = %s", str(admap))
self.platform_configs = {}
for platform_id, plat_objs in self.all_platforms.iteritems():
PLATFORM_CONFIG = {
'platform_id': platform_id,
'platform_topology': self.topology,
'agent_device_map': admap,
'agent_streamconfig_map': None, #self.agent_streamconfig_map,
'driver_config': DVR_CONFIG,
}
self.platform_configs[platform_id] = {
'platform_id': platform_id,
'platform_topology': self.topology,
'agent_device_map': admap,
'agent_streamconfig_map': self.agent_streamconfig_map,
'driver_config': DVR_CONFIG,
}
agent_config = {
'platform_config': PLATFORM_CONFIG,
}
self.stream_id = self.agent_streamconfig_map[platform_id]['stream_id']
# import pprint
# print '============== platform id within unit test: %s ===========' % platform_id
# pprint.pprint(agent_config)
#agent_config['platform_config']['agent_streamconfig_map'] = None
agent_instance_obj = IonObject(RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=agent_config)
agent_id = plat_objs['agent_id']
device_id = plat_objs['device_id']
agent_instance_id = self.imsclient.create_platform_agent_instance(
agent_instance_obj, agent_id, self.device_id)
plat_objs['agent_instance_obj'] = agent_instance_obj
plat_objs['agent_instance_id'] = agent_instance_id
stream_config = self.agent_streamconfig_map[platform_id]
self._start_data_subscriber(agent_instance_id, stream_config)
def _start_data_subscriber(self, stream_name, stream_config):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r', stream_name)
stream_id = self.stream_id #stream_config['stream_id']
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.pubsubcli.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.pubsubcli.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.pubsubcli.deactivate_subscription(sub.subscription_id)
except:
pass
self.pubsubcli.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
@skip("IMS does't net implement topology")
def test_hierarchy(self):
self._create_launch_verify(BASE_PLATFORM_ID)
@attr('INT', group='sa')
def test_single_platform(self):
self._create_launch_verify('LJ01D')
def _create_launch_verify(self, base_platform_id):
# and trigger the traversal of the branch rooted at that base platform
# to create corresponding ION objects and configuration dictionaries:
base_platform_objs = self._traverse(base_platform_id)
# now that most of the topology information is there, add the
# PlatformAgentInstance elements
self._set_platform_agent_instances()
base_platform_config = self.platform_configs[base_platform_id]
log.info("base_platform_id = %r", base_platform_id)
log.info("topology = %s", str(self.topology))
#-------------------------------------------------------------------------------------
# Create Data Process Definition and Data Process for the eng stream monitor process
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='DemoStreamAlertTransform',
description='For testing EventTriggeredTransform_B',
module='ion.processes.data.transforms.event_alert_transform',
class_name='DemoStreamAlertTransform')
self.platform_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#THERE SHOULD BE NO STREAMDEF REQUIRED HERE.
platform_streamdef_id = self.pubsubcli.create_stream_definition(name='platform_eng_parsed', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(platform_streamdef_id, self.platform_dprocdef_id, binding='output' )
config = {
'process':{
'timer_interval': 5,
'queue_name': 'a_queue',
'variable_name': 'input_voltage',
'time_field_name': 'preferred_timestamp',
'valid_values': [-100, 100],
'timer_origin': 'Interval Timer'
}
}
platform_data_process_id = self.dataprocessclient.create_data_process(self.platform_dprocdef_id, [self.data_product_id], {}, config)
self.dataprocessclient.activate_data_process(platform_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process, platform_data_process_id)
#-------------------------------
# Launch Base Platform AgentInstance, connect to the resource agent client
#-------------------------------
agent_instance_id = base_platform_objs['agent_instance_id']
pid = self.imsclient.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.imsclient.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
agent_instance_obj= self.imsclient.read_instrument_agent_instance(agent_instance_id)
log.debug('test_oms_create_and_launch: Platform agent instance obj: %s', str(agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient('paclient', name=agent_instance_obj.agent_process_id, process=FakeProcess())
log.debug(" test_oms_create_and_launch:: got pa client %s", str(self._pa_client))
log.debug("base_platform_config =\n%s", base_platform_config)
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug( 'Base Platform ping_agent = %s', str(retval) )
# issue INITIALIZE command to the base platform, which will launch the
# creation of the whole platform hierarchy rooted at base_platform_config['platform_id']
# cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE, kwargs=dict(plat_config=base_platform_config))
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform INITIALIZE = %s', str(retval) )
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_ACTIVE = %s', str(retval) )
# RUN: this command includes the launch of the resource monitoring greenlets
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RUN = %s', str(retval) )
# START_EVENT_DISPATCH
kwargs = dict(params="TODO set params")
cmd = AgentCommand(command=PlatformAgentEvent.START_EVENT_DISPATCH, kwargs=kwargs)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self.assertTrue(retval.result is not None)
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(15)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
#get the extended platfrom which wil include platform aggreate status fields
extended_platform = self.imsclient.get_platform_device_extension(self.device_id)
# log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) )
# log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) )
# log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) )
# STOP_EVENT_DISPATCH
cmd = AgentCommand(command=PlatformAgentEvent.STOP_EVENT_DISPATCH)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self.assertTrue(retval.result is not None)
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_INACTIVE = %s', str(retval) )
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RESET = %s', str(retval) )
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.imsclient.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
class TestIntDataProcessManagementServiceMultiOut(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
def test_createDataProcess(self):
#---------------------------------------------------------------------------
# Data Process Definition
#---------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# Make assertion on the newly registered data process definition
data_process_definition = self.rrclient.read(dprocdef_id)
self.assertEquals(data_process_definition.name, 'ctd_L0_all')
self.assertEquals(data_process_definition.description, 'transform ctd package into three separate L0 streams')
self.assertEquals(data_process_definition.module, 'ion.processes.data.transforms.ctd.ctd_L0_all')
self.assertEquals(data_process_definition.class_name, 'ctd_L0_all')
# Read the data process definition using data process management and make assertions
dprocdef_obj = self.dataprocessclient.read_data_process_definition(dprocdef_id)
self.assertEquals(dprocdef_obj.class_name,'ctd_L0_all')
self.assertEquals(dprocdef_obj.module,'ion.processes.data.transforms.ctd.ctd_L0_all')
#---------------------------------------------------------------------------
# Create an input instrument
#---------------------------------------------------------------------------
instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product')
instrument_id, rev = self.rrclient.create(instrument_obj)
# Register the instrument so that the data producer and stream object are created
data_producer_id = self.damsclient.register_instrument(instrument_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_input_stream_definition_to_data_process_definition(ctd_stream_def_id, dprocdef_id )
# Assert that the link between the stream definition and the data process definition was done
assocs = self.rrclient.find_associations(subject=dprocdef_id, predicate=PRED.hasInputStreamDefinition, object=ctd_stream_def_id, id_only=True)
self.assertIsNotNone(assocs)
#---------------------------------------------------------------------------
# Input Data Product
#---------------------------------------------------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_dp_obj = IonObject( RT.DataProduct,
name='InputDataProduct',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
input_dp_id = self.dataproductclient.create_data_product(data_product=input_dp_obj, stream_definition_id=ctd_stream_def_id, exchange_point='test')
#Make assertions on the input data product created
input_dp_obj = self.rrclient.read(input_dp_id)
self.assertEquals(input_dp_obj.name, 'InputDataProduct')
self.assertEquals(input_dp_obj.description, 'some new dp')
self.damsclient.assign_data_product(instrument_id, input_dp_id)
# Retrieve the stream via the DataProduct->Stream associations
stream_ids, _ = self.rrclient.find_objects(input_dp_id, PRED.hasStream, None, True)
self.in_stream_id = stream_ids[0]
#---------------------------------------------------------------------------
# Output Data Product
#---------------------------------------------------------------------------
outgoing_stream_conductivity_id = self.pubsubclient.create_stream_definition(name='conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_conductivity_id, dprocdef_id,binding='conductivity' )
outgoing_stream_pressure_id = self.pubsubclient.create_stream_definition(name='pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_pressure_id, dprocdef_id, binding='pressure' )
outgoing_stream_temperature_id = self.pubsubclient.create_stream_definition(name='temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_temperature_id, dprocdef_id, binding='temperature' )
self.output_products={}
output_dp_obj = IonObject(RT.DataProduct,
name='conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_1 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_conductivity_id)
self.output_products['conductivity'] = output_dp_id_1
output_dp_obj = IonObject(RT.DataProduct,
name='pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_2 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_pressure_id)
self.output_products['pressure'] = output_dp_id_2
output_dp_obj = IonObject(RT.DataProduct,
name='temperature',
description='transform output ',
temporal_domain = tdom,
spatial_domain = sdom)
output_dp_id_3 = self.dataproductclient.create_data_product(output_dp_obj, outgoing_stream_temperature_id)
self.output_products['temperature'] = output_dp_id_3
#---------------------------------------------------------------------------
# Create the data process
#---------------------------------------------------------------------------
def _create_data_process():
dproc_id = self.dataprocessclient.create_data_process(dprocdef_id, [input_dp_id], self.output_products)
return dproc_id
dproc_id = _create_data_process()
# Make assertions on the data process created
data_process = self.dataprocessclient.read_data_process(dproc_id)
# Assert that the data process has a process id attached
self.assertIsNotNone(data_process.process_id)
# Assert that the data process got the input data product's subscription id attached as its own input_susbcription_id attribute
self.assertIsNotNone(data_process.input_subscription_id)
output_data_product_ids = self.rrclient.find_objects(subject=dproc_id, predicate=PRED.hasOutputProduct, object_type=RT.DataProduct, id_only=True)
self.assertEquals(Set(output_data_product_ids[0]), Set([output_dp_id_1,output_dp_id_2,output_dp_id_3]))
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 60}}})
def test_createDataProcessUsingSim(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel" )
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver", driver_class="SBE37Driver" )
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
port_agent_config = {
'device_addr': 'sbe37-simulator.oceanobservatories.org',
'device_port': 4001,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'command_port': 4002,
'data_port': 4003,
'log_level': 5,
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", svr_addr="localhost",
comms_device_address=CFG.device.sbe37.host, comms_device_port=CFG.device.sbe37.port,
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE32_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='SBE37_RAW', parameter_dictionary_id=pdict_id)
dp_obj.name = 'ctd_raw'
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# Create listener for data process events and verify that events are received.
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-367 Data processing shall notify registered data product consumers about data processing workflow life cycle events
#todo (contd) ... I believe the capability does not exist yet now. ANS And SA are not yet publishing any workflow life cycle events (Swarbhanu)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
#-------------------------------
# Retrieve a list of all data process defintions in RR and validate that the DPD is listed
#-------------------------------
# todo: add this validate for Req: L4-CI-SA-RQ-366 Data processing shall manage data topic definitions
# todo: This capability is not yet completed (Swarbhanu)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
#todo: check that activate event is received L4-CI-SA-RQ-367
#todo... (it looks like no event is being published when the data process is activated... so below, we just check for now
# todo... that the subscription is indeed activated) (Swarbhanu)
# todo: monitor process to see if it is active (sa-rq-182)
ctd_l0_all_data_process = self.rrclient.read(ctd_l0_all_data_process_id)
input_subscription_id = ctd_l0_all_data_process.input_subscription_id
subs = self.rrclient.read(input_subscription_id)
self.assertTrue(subs.activated)
# todo: This has not yet been completed by CEI, will prbly surface thru a DPMS call
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the extended resources for data process definition and for data process
#-------------------------------
extended_process_definition = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
self.assertEqual(1, len(extended_process_definition.data_processes))
log.debug("test_createDataProcess: extended_process_definition %s", str(extended_process_definition))
extended_process = self.dataprocessclient.get_data_process_extension(ctd_l0_all_data_process_id)
self.assertEqual(1, len(extended_process.input_data_products))
log.debug("test_createDataProcess: extended_process %s", str(extended_process))
#-------------------------------
# Cleanup
#-------------------------------
self.dataprocessclient.delete_data_process(ctd_l0_all_data_process_id)
self.dataprocessclient.delete_data_process_definition(ctd_L0_all_dprocdef_id)
self.dataprocessclient.force_delete_data_process(ctd_l0_all_data_process_id)
self.dataprocessclient.force_delete_data_process_definition(ctd_L0_all_dprocdef_id)
class TestDataProcessWithLookupTable(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.dataset_management = self.datasetclient
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
def test_lookupTableProcessing(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
log.info(
'test_createTransformsThenActivateInstrument: new InstrumentModel id = %s',
instModel_id)
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD)
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.info(
'test_createTransformsThenActivateInstrument: new InstrumentAgent id = %s',
instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice and attachment for lookup table
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
try:
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.info(
'test_createTransformsThenActivateInstrument: new InstrumentDevice id = %s',
instDevice_id)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment,
name='deviceLookupTable',
content=base64.encodestring(contents),
keywords=['DataProcessInput'],
attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(instDevice_id, att)
log.info(
'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s',
deviceAttachment)
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance")
self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
name='SBE37_CDM', parameter_dictionary_id=pdict_id)
log.info(
'TestDataProcessWithLookupTable: new Stream Definition id = %s',
instDevice_id)
log.info('Creating new CDM data product with a stream definition')
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(
dp_obj, ctd_stream_def_id)
log.info('new ctd_parsed_data_product_id = %s',
ctd_parsed_data_product)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product,
PRED.hasStream, None, True)
log.info('TestDataProcessWithLookupTable: Data product streams1 = %s',
stream_ids)
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
log.info(
'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition'
)
raw_stream_def_id = self.pubsubclient.create_stream_definition(
name='SBE37_RAW', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='ctd_raw',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(
dp_obj, raw_stream_def_id)
log.info('new ctd_raw_data_product_id = %s', ctd_raw_data_product)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product,
PRED.hasStream, None, True)
log.info('Data product streams2 = %s', stream_ids)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug(
"TestDataProcessWithLookupTable: create data process definition ctd_L0_all"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new ctd_L0_all data process definition: %s" %
ex)
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..."
att = IonObject(RT.Attachment,
name='processDefinitionLookupTable',
content=base64.encodestring(contents),
keywords=['DataProcessInput'],
attachment_type=AttachmentType.ASCII)
processDefinitionAttachment = self.rrclient.create_attachment(
ctd_L0_all_dprocdef_id, att)
log.debug(
"TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s",
str(processDefinitionAttachment))
processDefinitionAttachment_obj = self.rrclient.read(
processDefinitionAttachment)
log.debug(
"TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s",
str(processDefinitionAttachment_obj))
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id,
ctd_L0_all_dprocdef_id,
binding='conductivity')
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id,
ctd_L0_all_dprocdef_id,
binding='pressure')
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id,
ctd_L0_all_dprocdef_id,
binding='temperature')
log.debug(
"TestDataProcessWithLookupTable: create output data product L0 conductivity"
)
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
log.debug(
"TestDataProcessWithLookupTable: create output data product L0 pressure"
)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
log.debug(
"TestDataProcessWithLookupTable: create output data product L0 temperature"
)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain=tdom,
spatial_domain=sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug(
"TestDataProcessWithLookupTable: create L0 all data_process start")
try:
in_prods = []
in_prods.append(ctd_parsed_data_product)
out_prods = [
ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id,
ctd_l0_temperature_output_dp_id
]
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=ctd_L0_all_dprocdef_id,
in_data_product_ids=in_prods,
out_data_product_ids=out_prods)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"TestDataProcessWithLookupTable: create L0 all data_process return"
)
data_process = self.rrclient.read(ctd_l0_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=ctd_l0_all_data_process_id,
predicate=PRED.hasProcess,
object_type=RT.Process,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..."
att = IonObject(RT.Attachment,
name='processLookupTable',
content=base64.encodestring(contents),
keywords=['DataProcessInput'],
attachment_type=AttachmentType.ASCII)
processAttachment = self.rrclient.create_attachment(
ctd_l0_all_data_process_id, att)
log.info(
'TestDataProcessWithLookupTable: InstrumentDevice attachment id = %s',
processAttachment)
class TestCTDTransformsIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def test_createTransformsThenActivateInstrument(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance",
driver_module='mi.instrument.seabird.sbe37smb.ooicore.driver', driver_class='SBE37Driver',
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
log.debug( 'new Stream Definition id = %s', instDevice_id)
log.debug( 'Creating new CDM data product with a stream definition')
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
log.debug( 'Creating new RAW data product with a stream definition')
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
log.debug( 'new dp_id = %s', str(ctd_raw_data_product))
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
log.debug( 'Data product streams2 = %s', str(stream_ids))
#todo: attaching the taxonomy to the stream is a TEMPORARY measure
# Create taxonomies for both parsed and attach to the stream
RawTax = TaxyTool()
RawTax.add_taxonomy_set('raw_fixed','Fixed length bytes in an array of records')
RawTax.add_taxonomy_set('raw_blob','Unlimited length bytes in an array')
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id,
parameter_dictionary)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id,
parameter_dictionary)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id,
parameter_dictionary)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id )
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id )
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id,
parameter_dictionary
)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_pressure', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_temperature', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id )
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_salinity', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density")
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_density', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return")
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return")
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Streaming
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: initialize %s", str(retval))
log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state))
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from run state: %s", str(state))
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG : 1,
SBE37Parameter.INTERVAL : 5,
SBE37Parameter.TXREALTIME : True
}
self._ia_client.set_param(params)
log.debug("test_activateInstrumentStream: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state))
time.sleep(7)
log.debug("test_activateInstrumentStream: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
log.debug("test_activateInstrumentStream: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result))
time.sleep(2)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestCTDTransformsIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
def test_createTransformsThenActivateInstrument(self):
# Set up the preconditions
# ingestion configuration parameters
self.exchange_point_id = 'science_data'
self.number_of_workers = 2
self.hdf_storage = HdfStorage(relative_path='ingest')
self.couch_storage = CouchStorage(datastore_name='test_datastore')
self.XP = 'science_data'
self.exchange_name = 'ingestion_queue'
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
model_label="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent")
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
try:
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
# driver_config = {
# 'svr_addr': "localhost",
# 'cmd_port': 5556,
# 'evt_port': 5557,
# 'dvr_mod': "ion.agents.instrument.drivers.sbe37.sbe37_driver",
# 'dvr_cls': "SBE37Driver",
# 'comms_config': {
# 'addr': 'sbe37-simulator.oceanobservatories.org',
# 'port': 4001,
# }
# }
driver_config = {
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'workdir': '/tmp/',
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org',
comms_device_port=4001,
port_agent_work_dir='/tmp/',
port_agent_delimeter=['<<', '>>'])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
container=ctd_stream_def)
print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(
dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_parsed_data_product,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product,
PRED.hasStream, None, True)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(
container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='ctd_raw',
description='raw stream test')
try:
ctd_raw_data_product = self.dataproductclient.create_data_product(
dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_raw_data_product,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product,
PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new ctd_L0_all data process definition: %s" %
ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1ConductivityTransform data process definition: %s"
% ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1PressureTransform data process definition: %s"
% ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new CTDL1TemperatureTransform data process definition: %s"
% ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new SalinityTransform data process definition: %s"
% ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug(
"TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform"
)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except BadRequest as ex:
self.fail(
"failed to create new DensityTransform data process definition: %s"
% ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id)
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id)
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id)
self.output_products = {}
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L0 conductivity"
)
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity')
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l0_conductivity_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L0 pressure"
)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Pressure',
description='transform output pressure')
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l0_pressure_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L0 temperature"
)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Temperature',
description='transform output temperature')
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l0_temperature_output_dp_id,
persist_data=True,
persist_metadata=True)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_conductivity_id,
ctd_L1_conductivity_dprocdef_id)
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id)
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L1 conductivity"
)
ctd_l1_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity')
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l1_conductivity_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L1 pressure"
)
ctd_l1_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure')
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l1_pressure_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L1 temperature"
)
ctd_l1_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature')
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l1_temperature_output_dp_id,
persist_data=True,
persist_metadata=True)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id)
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(
container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L2 Salinity"
)
ctd_l2_salinity_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity')
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l2_salinity_output_dp_id,
persist_data=True,
persist_metadata=True)
log.debug(
"test_createTransformsThenActivateInstrument: create output data product L2 Density"
)
ctd_l2_density_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Density',
description='transform output pressure')
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_l2_density_output_dp_id,
persist_data=True,
persist_metadata=True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L0 all data_process start"
)
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [ctd_parsed_data_product],
self.output_products)
self.dataprocessclient.activate_data_process(
ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L0 all data_process return"
)
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start"
)
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
ctd_L1_conductivity_dprocdef_id,
[ctd_l0_conductivity_output_dp_id],
{'output': ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(
l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return"
)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process start"
)
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id],
{'output': ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(
l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process return"
)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process start"
)
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L1_temperature_dprocdef_id,
[ctd_l0_temperature_output_dp_id],
{'output': ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(
l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L1_Pressure data_process return"
)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L2_salinity data_process start"
)
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product],
{'output': ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(
l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L2_salinity data_process return"
)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug(
"test_createTransformsThenActivateInstrument: create L2_Density data_process start"
)
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L2_density_dprocdef_id, [ctd_parsed_data_product],
{'output': ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(
l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug(
"test_createTransformsThenActivateInstrument: create L2_Density data_process return"
)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
'iaclient',
name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug(
" test_createTransformsThenActivateInstrument:: got ia client %s",
str(self._ia_client))
#-------------------------------
# Streaming
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
print retval
log.debug(
"test_createTransformsThenActivateInstrument:: initialize %s",
str(retval))
time.sleep(1)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: go_active %s",
str(reply))
time.sleep(1)
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_createTransformsThenActivateInstrument:: run %s",
str(reply))
time.sleep(1)
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG: 1,
SBE37Parameter.INTERVAL: 5,
SBE37Parameter.TXREALTIME: True
}
self._ia_client.set_param(params)
self._no_samples = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute_agent(cmd)
log.debug(
"test_createTransformsThenActivateInstrument:: go_streaming %s",
str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(15)
log.debug("test_activateInstrument: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrument: return from go_observatory %s",
str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
time.sleep(2)
log.debug(
"test_createTransformsThenActivateInstrument:: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug(
"test_createTransformsThenActivateInstrument:: return from reset %s",
str(reply))
time.sleep(2)
#-------------------------------
# Sampling
#-------------------------------
# cmd = AgentCommand(command='initialize')
# retval = self._ia_client.execute_agent(cmd)
# print retval
# log.debug("test_createTransformsThenActivateInstrument:: initialize %s", str(retval))
# time.sleep(2)
#
# cmd = AgentCommand(command='go_active')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: go_active %s", str(reply))
# time.sleep(2)
#
# cmd = AgentCommand(command='run')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: run %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample ")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 2")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 2 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling acquire_sample 3")
# cmd = AgentCommand(command='acquire_sample')
# reply = self._ia_client.execute(cmd)
# log.debug("test_activateInstrument: return from acquire_sample 3 %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling go_inactive ")
# cmd = AgentCommand(command='go_inactive')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from go_inactive %s", str(reply))
# time.sleep(2)
#
# log.debug("test_activateInstrument: calling reset ")
# cmd = AgentCommand(command='reset')
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activateInstrument: return from reset %s", str(reply))
# time.sleep(2)
#
#
# self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
#
#
# #get the dataset id of the ctd_parsed product from the dataproduct ctd_parsed_data_product
# ctd_parsed_data_product_obj = self.dataproductclient.read_data_product(ctd_parsed_data_product)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset id %s", str(ctd_parsed_data_product_obj.dataset_id))
#
# # ask for the dataset bounds from the datasetmgmtsvc
# bounds = self.datasetclient.get_dataset_bounds(ctd_parsed_data_product_obj.dataset_id)
# log.debug("test_createTransformsThenActivateInstrument:: ctd_parsed_data_product dataset bounds %s", str(bounds))
# print 'activate_instrument: got dataset bounds %s', str(bounds)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
