def setUp(self):
# Start container
logging.disable(logging.ERROR)
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# simulate preloading
preload_ion_params(self.container)
logging.disable(logging.NOTSET)
#Instantiate a process to represent the test
process=VisualizationServiceTestProcess()
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process)
self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process)
self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process)
self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process)
self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process)
self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process)
self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process)
self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process)
self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process)
self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process)
self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# simulate preloading
preload_ion_params(self.container)
#Instantiate a process to represent the test
process=WorkflowServiceTestProcess()
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process)
self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process)
self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process)
self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process)
self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process)
self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process)
self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process)
self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process)
self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process)
self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def load_parameter_function(self, row):
name = row['Name']
ftype = row['Function Type']
func_expr = row['Function']
owner = row['Owner']
args = ast.literal_eval(row['Args'])
#kwargs = row['Kwargs']
descr = row['Description']
data_process_management = DataProcessManagementServiceProcessClient(
self)
function_type = None
if ftype == 'PythonFunction':
function_type = PFT.PYTHON
elif ftype == 'NumexprFunction':
function_type = PFT.NUMEXPR
else:
raise Conflict('Unsupported Function Type: %s' % ftype)
parameter_function = ParameterFunctionResource(
name=name,
function=func_expr,
function_type=function_type,
owner=owner,
args=args,
description=descr)
parameter_function.alt_ids = ['PRE:' + row['ID']]
parameter_function_id = self.create_parameter_function(
parameter_function)
dpd = DataProcessDefinition()
dpd.name = name
dpd.description = 'Parameter Function Definition for %s' % name
dpd.data_process_type = DataProcessTypeEnum.PARAMETER_FUNCTION
dpd.parameters = args
data_process_management.create_data_process_definition(
dpd, parameter_function_id)
return parameter_function_id
def load_parameter_function(self, row):
name = row['Name']
ftype = row['Function Type']
func_expr = row['Function']
owner = row['Owner']
args = ast.literal_eval(row['Args'])
#kwargs = row['Kwargs']
descr = row['Description']
data_process_management = DataProcessManagementServiceProcessClient(self)
function_type=None
if ftype == 'PythonFunction':
function_type = PFT.PYTHON
elif ftype == 'NumexprFunction':
function_type = PFT.NUMEXPR
else:
raise Conflict('Unsupported Function Type: %s' % ftype)
parameter_function = ParameterFunctionResource(
name=name,
function=func_expr,
function_type=function_type,
owner=owner,
args=args,
description=descr)
parameter_function.alt_ids = ['PRE:' + row['ID']]
parameter_function_id = self.create_parameter_function(parameter_function)
dpd = DataProcessDefinition()
dpd.name = name
dpd.description = 'Parameter Function Definition for %s' % name
dpd.data_process_type = DataProcessTypeEnum.PARAMETER_FUNCTION
dpd.parameters = args
data_process_management.create_data_process_definition(dpd, parameter_function_id)
return parameter_function_id
class TestWorkflowManagementIntegration(VisualizationIntegrationTestHelper):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
#Instantiate a process to represent the test
process=WorkflowServiceTestProcess()
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process)
self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process)
self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process)
self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process)
self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process)
self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process)
self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process)
self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process)
self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process)
self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process)
self.ctd_stream_def = SBE37_CDM_stream_definition()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_SA_transform_components(self):
assertions = self.assertTrue
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product()
data_product_stream_ids.append(ctd_stream_id)
###
### Setup the first transformation
###
# Salinity: Data Process Definition
ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition()
l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process(ctd_L2_salinity_dprocdef_id,ctd_parsed_data_product_id, 'salinity' )
## get the stream id for the transform outputs
stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
sal_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_stream_id)
###
### Setup the second transformation
###
# Salinity Doubler: Data Process Definition
salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition()
salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process(salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id, 'salinity' )
stream_ids, _ = self.rrclient.find_objects(salinity_doubler_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
sal_dbl_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_dbl_stream_id)
#Start the output stream listener to monitor and collect messages
results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids)
#Stop the transform processes
self.dataprocessclient.deactivate_data_process(salinity_double_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
#Validate the data from each of the messages along the way
self.validate_messages(results)
@attr('LOCOINT')
@attr('SMOKE')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_transform_workflow(self):
assertions = self.assertTrue
log.debug("Building the workflow definition")
workflow_def_obj = IonObject(RT.WorkflowDefinition,
name='Salinity_Test_Workflow',
description='tests a workflow of multiple transform data processes')
workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name
#-------------------------------------------------------------------------------------------------------------------------
log.debug( "Adding a transformation process definition for salinity")
#-------------------------------------------------------------------------------------------------------------------------
ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep',
data_process_definition_id=ctd_L2_salinity_dprocdef_id,
persist_process_output_data=False) #Don't persist the intermediate data product
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#-------------------------------------------------------------------------------------------------------------------------
log.debug( "Adding a transformation process definition for salinity doubler")
#-------------------------------------------------------------------------------------------------------------------------
salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep',
data_process_definition_id=salinity_doubler_dprocdef_id, )
workflow_def_obj.workflow_steps.append(workflow_step_obj)
log.debug( "Creating workflow def in the resource registry")
workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)
aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition)
assertions(len(aids) == 2 )
#The list of data product streams to monitor
data_product_stream_ids = list()
log.debug( "Creating the input data product")
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product()
data_product_stream_ids.append(ctd_stream_id)
log.debug( "Creating and starting the workflow")
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id,
ctd_parsed_data_product_id,
persist_workflow_data_product=True, output_data_product_name=workflow_data_product_name, timeout=300)
workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1 )
log.debug( "persisting the output product")
#self.dataproductclient.activate_data_product_persistence(workflow_product_id)
dataset_ids,_ = self.rrclient.find_objects(workflow_product_id, PRED.hasDataset, RT.Dataset, True)
assertions(len(dataset_ids) == 1 )
dataset_id = dataset_ids[0]
log.debug( "Verifying the output data product name matches what was specified in the workflow definition")
workflow_product = self.rrclient.read(workflow_product_id)
assertions(workflow_product.name.startswith(workflow_data_product_name), 'Nope: %s != %s' % (workflow_product.name, workflow_data_product_name))
log.debug( "Walking the associations to find the appropriate output data streams to validate the messages")
workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 2 )
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) == 1 )
data_product_stream_ids.append(stream_ids[0])
log.debug( "data_product_stream_ids: %s" % data_product_stream_ids)
log.debug( "Starting the output stream listener to monitor to collect messages")
results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids)
log.debug( "results::: %s" % results)
log.debug( "Stopping the workflow processes")
self.workflowclient.terminate_data_process_workflow(workflow_id, False, timeout=250) # Should test true at some point
log.debug( "Making sure the Workflow object was removed")
objs, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(objs) == 0)
log.debug( "Validating the data from each of the messages along the way")
self.validate_messages(results)
log.debug( "Checking to see if dataset id = %s, was persisted, and that it can be retrieved...." % dataset_id)
self.validate_data_ingest_retrieve(dataset_id)
log.debug( "Cleaning up to make sure delete is correct.")
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_google_dt_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='GoogleDT_Test_Workflow',description='Tests the workflow of converting stream data to Google DT')
#Add a transformation process definition
google_dt_procdef_id = self.create_google_dt_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id, persist_process_output_data=False)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product()
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, timeout=60)
workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1 )
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1 )
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) == 1 )
data_product_stream_ids.append(stream_ids[0])
#Start the output stream listener to monitor and collect messages
results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id,delete_data_products=False, timeout=60) # Should test true at some point
#Validate the data from each of the messages along the way
self.validate_google_dt_transform_results(results)
"""
# Check to see if ingestion worked. Extract the granules from data_retrieval.
# First find the dataset associated with the output dp product
ds_ids,_ = self.rrclient.find_objects(workflow_dp_ids[len(workflow_dp_ids) - 1], PRED.hasDataset, RT.Dataset, True)
retrieved_granule = self.data_retriever.retrieve(ds_ids[0])
#Validate the data from each of the messages along the way
self.validate_google_dt_transform_results(retrieved_granule)
"""
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_mpl_graphs_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Mpl_Graphs_Test_Workflow',description='Tests the workflow of converting stream data to Matplotlib graphs')
#Add a transformation process definition
mpl_graphs_procdef_id = self.create_mpl_graphs_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=mpl_graphs_procdef_id, persist_process_output_data=False)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product()
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id,
persist_workflow_data_product=True, timeout=60)
workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1 )
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1 )
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) == 1 )
data_product_stream_ids.append(stream_ids[0])
#Start the output stream listener to monitor and collect messages
results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id,delete_data_products=False, timeout=60) # Should test true at some point
#Validate the data from each of the messages along the way
self.validate_mpl_graphs_transform_results(results)
# Check to see if ingestion worked. Extract the granules from data_retrieval.
# First find the dataset associated with the output dp product
ds_ids,_ = self.rrclient.find_objects(workflow_dp_ids[len(workflow_dp_ids) - 1], PRED.hasDataset, RT.Dataset, True)
retrieved_granule = self.data_retriever.retrieve_last_data_points(ds_ids[0], 10)
#Validate the data from each of the messages along the way
self.validate_mpl_graphs_transform_results(retrieved_granule)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_multiple_workflow_instances(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Multiple_Test_Workflow',description='Tests the workflow of converting stream data')
#Add a transformation process definition
google_dt_procdef_id = self.create_google_dt_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id, persist_process_output_data=False)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the first input data product
ctd_stream_id1, ctd_parsed_data_product_id1 = self.create_ctd_input_stream_and_data_product('ctd_parsed1')
data_product_stream_ids.append(ctd_stream_id1)
#Create and start the first workflow
workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id1, timeout=60)
#Create the second input data product
ctd_stream_id2, ctd_parsed_data_product_id2 = self.create_ctd_input_stream_and_data_product('ctd_parsed2')
data_product_stream_ids.append(ctd_stream_id2)
#Create and start the second workflow
workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id2, timeout=60)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_ids,_ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 2 )
#Start the first input stream process
ctd_sim_pid1 = self.start_sinusoidal_input_stream_process(ctd_stream_id1)
#Start the second input stream process
ctd_sim_pid2 = self.start_simple_input_stream_process(ctd_stream_id2)
#Start the output stream listener to monitor a set number of messages being sent through the workflows
results = self.start_output_stream_and_listen(None, data_product_stream_ids, message_count_per_stream=5)
# stop the flow of messages...
self.process_dispatcher.cancel_process(ctd_sim_pid1) # kill the ctd simulator process - that is enough data
self.process_dispatcher.cancel_process(ctd_sim_pid2)
#Stop the first workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id1,delete_data_products=False, timeout=60) # Should test true at some point
#Stop the second workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id2,delete_data_products=False, timeout=60) # Should test true at some point
workflow_ids,_ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 0 )
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
aid_list = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition)
assertions(len(aid_list) == 0 )
class TestWorkflowManagementIntegration(VisualizationIntegrationTestHelper):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# simulate preloading
preload_ion_params(self.container)
#Instantiate a process to represent the test
process = WorkflowServiceTestProcess()
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceProcessClient(
node=self.container.node, process=process)
self.damsclient = DataAcquisitionManagementServiceProcessClient(
node=self.container.node, process=process)
self.pubsubclient = PubsubManagementServiceProcessClient(
node=self.container.node, process=process)
self.ingestclient = IngestionManagementServiceProcessClient(
node=self.container.node, process=process)
self.imsclient = InstrumentManagementServiceProcessClient(
node=self.container.node, process=process)
self.dataproductclient = DataProductManagementServiceProcessClient(
node=self.container.node, process=process)
self.dataprocessclient = DataProcessManagementServiceProcessClient(
node=self.container.node, process=process)
self.datasetclient = DatasetManagementServiceProcessClient(
node=self.container.node, process=process)
self.workflowclient = WorkflowManagementServiceProcessClient(
node=self.container.node, process=process)
self.process_dispatcher = ProcessDispatcherServiceProcessClient(
node=self.container.node, process=process)
self.data_retriever = DataRetrieverServiceProcessClient(
node=self.container.node, process=process)
self.ctd_stream_def = SBE37_CDM_stream_definition()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
def test_SA_transform_components(self):
assertions = self.assertTrue
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
###
### Setup the first transformation
###
# Salinity: Data Process Definition
ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition(
)
l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process(
ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product_id,
'salinity')
## get the stream id for the transform outputs
stream_ids, _ = self.rrclient.find_objects(
ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0)
sal_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_stream_id)
###
### Setup the second transformation
###
# Salinity Doubler: Data Process Definition
salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition(
)
salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process(
salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id,
'salinity')
stream_ids, _ = self.rrclient.find_objects(
salinity_doubler_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0)
sal_dbl_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_dbl_stream_id)
#Start the output stream listener to monitor and collect messages
results = self.start_output_stream_and_listen(ctd_stream_id,
data_product_stream_ids)
#Stop the transform processes
self.dataprocessclient.deactivate_data_process(
salinity_double_data_process_id)
self.dataprocessclient.deactivate_data_process(
l2_salinity_all_data_process_id)
#Validate the data from each of the messages along the way
self.validate_messages(results)
@attr('LOCOINT')
@attr('SMOKE')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
def test_transform_workflow(self):
assertions = self.assertTrue
log.debug("Building the workflow definition")
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='Salinity_Test_Workflow',
description='tests a workflow of multiple transform data processes'
)
workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name
#-------------------------------------------------------------------------------------------------------------------------
log.debug("Adding a transformation process definition for salinity")
#-------------------------------------------------------------------------------------------------------------------------
ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=ctd_L2_salinity_dprocdef_id,
persist_process_output_data=False
) #Don't persist the intermediate data product
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#-------------------------------------------------------------------------------------------------------------------------
log.debug(
"Adding a transformation process definition for salinity doubler")
#-------------------------------------------------------------------------------------------------------------------------
salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=salinity_doubler_dprocdef_id,
)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
log.debug("Creating workflow def in the resource registry")
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
aids = self.rrclient.find_associations(workflow_def_id,
PRED.hasDataProcessDefinition)
assertions(len(aids) == 2)
#The list of data product streams to monitor
data_product_stream_ids = list()
log.debug("Creating the input data product")
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
log.debug("Creating and starting the workflow")
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(
workflow_def_id,
ctd_parsed_data_product_id,
persist_workflow_data_product=True,
output_data_product_name=workflow_data_product_name,
timeout=300)
workflow_output_ids, _ = self.rrclient.find_subjects(
RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1)
log.debug("persisting the output product")
#self.dataproductclient.activate_data_product_persistence(workflow_product_id)
dataset_ids, _ = self.rrclient.find_objects(workflow_product_id,
PRED.hasDataset,
RT.Dataset, True)
assertions(len(dataset_ids) == 1)
dataset_id = dataset_ids[0]
log.debug(
"Verifying the output data product name matches what was specified in the workflow definition"
)
workflow_product = self.rrclient.read(workflow_product_id)
assertions(
workflow_product.name.startswith(workflow_data_product_name),
'Nope: %s != %s' %
(workflow_product.name, workflow_data_product_name))
log.debug(
"Walking the associations to find the appropriate output data streams to validate the messages"
)
workflow_dp_ids, _ = self.rrclient.find_objects(
workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 2)
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
None, True)
assertions(len(stream_ids) == 1)
data_product_stream_ids.append(stream_ids[0])
log.debug("data_product_stream_ids: %s" % data_product_stream_ids)
log.debug(
"Starting the output stream listener to monitor to collect messages"
)
results = self.start_output_stream_and_listen(ctd_stream_id,
data_product_stream_ids)
log.debug("results::: %s" % results)
log.debug("Stopping the workflow processes")
self.workflowclient.terminate_data_process_workflow(
workflow_id, False, timeout=250) # Should test true at some point
log.debug("Making sure the Workflow object was removed")
objs, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(objs) == 0)
log.debug(
"Validating the data from each of the messages along the way")
self.validate_messages(results)
log.debug(
"Checking to see if dataset id = %s, was persisted, and that it can be retrieved...."
% dataset_id)
self.validate_data_ingest_retrieve(dataset_id)
log.debug("Cleaning up to make sure delete is correct.")
self.workflowclient.delete_workflow_definition(workflow_def_id)
"""
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
"""
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
def test_highcharts_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='HighCharts_Test_Workflow',
description=
'Tests the workflow of converting stream data to HighCharts')
#Add a transformation process definition
highcharts_procdef_id = self.create_highcharts_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=highcharts_procdef_id,
persist_process_output_data=False)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id, timeout=60)
workflow_output_ids, _ = self.rrclient.find_subjects(
RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids, _ = self.rrclient.find_objects(
workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1)
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
None, True)
assertions(len(stream_ids) == 1)
data_product_stream_ids.append(stream_ids[0])
#Start the output stream listener to monitor and collect messages
results = self.start_output_stream_and_listen(ctd_stream_id,
data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id=workflow_id, delete_data_products=False,
timeout=60) # Should test true at some point
#Validate the data from each of the messages along the way
self.validate_highcharts_transform_results(results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
"""
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
"""
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
def test_mpl_graphs_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='Mpl_Graphs_Test_Workflow',
description=
'Tests the workflow of converting stream data to Matplotlib graphs'
)
#Add a transformation process definition
mpl_graphs_procdef_id = self.create_mpl_graphs_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=mpl_graphs_procdef_id,
persist_process_output_data=False)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(
workflow_def_id,
ctd_parsed_data_product_id,
persist_workflow_data_product=True,
timeout=60)
workflow_output_ids, _ = self.rrclient.find_subjects(
RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids, _ = self.rrclient.find_objects(
workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1)
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
None, True)
assertions(len(stream_ids) == 1)
data_product_stream_ids.append(stream_ids[0])
#Start the output stream listener to monitor and collect messages
results = self.start_output_stream_and_listen(ctd_stream_id,
data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id=workflow_id, delete_data_products=False,
timeout=60) # Should test true at some point
#Validate the data from each of the messages along the way
self.validate_mpl_graphs_transform_results(results)
# Check to see if ingestion worked. Extract the granules from data_retrieval.
# First find the dataset associated with the output dp product
ds_ids, _ = self.rrclient.find_objects(
workflow_dp_ids[len(workflow_dp_ids) - 1], PRED.hasDataset,
RT.Dataset, True)
retrieved_granule = self.data_retriever.retrieve_last_data_points(
ds_ids[0], 10)
#Validate the data from each of the messages along the way
self.validate_mpl_graphs_transform_results(retrieved_granule)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
"""
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0 )
"""
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
def test_multiple_workflow_instances(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='Multiple_Test_Workflow',
description='Tests the workflow of converting stream data')
#Add a transformation process definition
highcharts_procdef_id = self.create_highcharts_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=highcharts_procdef_id,
persist_process_output_data=False)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the first input data product
ctd_stream_id1, ctd_parsed_data_product_id1 = self.create_ctd_input_stream_and_data_product(
'ctd_parsed1')
data_product_stream_ids.append(ctd_stream_id1)
#Create and start the first workflow
workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id1, timeout=60)
#Create the second input data product
ctd_stream_id2, ctd_parsed_data_product_id2 = self.create_ctd_input_stream_and_data_product(
'ctd_parsed2')
data_product_stream_ids.append(ctd_stream_id2)
#Create and start the second workflow
workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id2, timeout=60)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 2)
#Start the first input stream process
ctd_sim_pid1 = self.start_sinusoidal_input_stream_process(
ctd_stream_id1)
#Start the second input stream process
ctd_sim_pid2 = self.start_simple_input_stream_process(ctd_stream_id2)
#Start the output stream listener to monitor a set number of messages being sent through the workflows
results = self.start_output_stream_and_listen(
None, data_product_stream_ids, message_count_per_stream=5)
# stop the flow of messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid1
) # kill the ctd simulator process - that is enough data
self.process_dispatcher.cancel_process(ctd_sim_pid2)
#Stop the first workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id=workflow_id1, delete_data_products=False,
timeout=60) # Should test true at some point
#Stop the second workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id=workflow_id2, delete_data_products=False,
timeout=60) # Should test true at some point
workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 0)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
"""
