def test_create_workflow(self):
"""
Test that a workflow with all the associated workflow steps is actually created.
"""
workflow_step_list = [
TestFactory.create_workflow_step(
"tvb.tests.framework.adapters.testadapter2",
"TestAdapter2",
step_index=1,
static_kwargs={"test2": 2}),
TestFactory.create_workflow_step(
"tvb.tests.framework.adapters.testadapter1",
"TestAdapter1",
step_index=2,
static_kwargs={
"test1_val1": 1,
"test1_val2": 1
})
]
burst_id = self.__create_complex_workflow(workflow_step_list)
workflow_entities = dao.get_workflows_for_burst(burst_id)
self.assertTrue(
len(workflow_entities) == 1,
"For some reason workflow was not stored in database.")
workflow_steps = dao.get_workflow_steps(workflow_entities[0].id)
self.assertEqual(len(workflow_steps),
len(workflow_step_list) + 1,
"Wrong number of workflow steps created.")
def test_workflow_dynamic_params(self):
"""
A simple test just for the fact that dynamic parameters are passed properly
between two workflow steps:
step1 - tvb.tests.framework.adapters.testadapter1.TestAdapter1
step2 - tvb.tests.framework.adapters.testadapter3.TestAdapter3
The first adapter returns a tvb.datatypes.datatype1.Datatype1 instance.
The second adapter has this passed as a dynamic workflow parameter.
We check that the steps are actually ran by checking that two operations
are created and that two dataTypes are stored.
"""
workflow_step_list = [TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter1",
"TestAdapter1", step_index=1,
static_kwargs={"test1_val1": 1, "test1_val2": 1}),
TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter3",
"TestAdapter3", step_index=2,
dynamic_kwargs={
"test": {wf_cfg.DATATYPE_INDEX_KEY: 0,
wf_cfg.STEP_INDEX_KEY: 1}})]
self.__create_complex_workflow(workflow_step_list)
stored_datatypes = dao.get_datatypes_in_project(self.test_project.id)
self.assertTrue(len(stored_datatypes) == 3, "DataType from all step were not stored.")
for result_row in stored_datatypes:
self.assertTrue(result_row.type in ['Datatype1', 'Datatype2'], "Wrong type was stored.")
finished, started, error, _, _ = dao.get_operation_numbers(self.test_project.id)
self.assertEqual(finished, 3, "Didn't start operations for both adapters in workflow.")
self.assertEqual(started, 0, "Some operations from workflow didn't finish.")
self.assertEqual(error, 0, "Some operations finished with error status.")
def test_workflow_generation(self):
"""
A simple test just for the fact that a workflow is created an ran,
no dynamic parameters are passed. In this case we create a two steps
workflow: step1 - tvb.tests.framework.adapters.testadapter2.TestAdapter2
step2 - tvb.tests.framework.adapters.testadapter1.TestAdapter1
The first adapter doesn't return anything and the second returns one
tvb.datatypes.datatype1.Datatype1 instance. We check that the steps
are actually ran by checking that two operations are created and that
one dataType is stored.
"""
workflow_step_list = [TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter2",
"TestAdapter2", step_index=1,
static_kwargs={"test2": 2}),
TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter1",
"TestAdapter1", step_index=2,
static_kwargs={"test1_val1": 1, "test1_val2": 1})]
self.__create_complex_workflow(workflow_step_list)
stored_datatypes = dao.get_datatypes_in_project(self.test_project.id)
self.assertTrue(len(stored_datatypes) == 2, "DataType from second step was not stored.")
self.assertTrue(stored_datatypes[0].type == 'Datatype1', "Wrong type was stored.")
self.assertTrue(stored_datatypes[1].type == 'Datatype1', "Wrong type was stored.")
finished, started, error, _, _ = dao.get_operation_numbers(self.test_project.id)
self.assertEqual(finished, 3, "Didnt start operations for both adapters in workflow.")
self.assertEqual(started, 0, "Some operations from workflow didnt finish.")
self.assertEqual(error, 0, "Some operations finished with error status.")
def test_workflow_dynamic_params(self):
"""
A simple test just for the fact that dynamic parameters are passed properly
between two workflow steps:
step1 - tvb.tests.framework.adapters.testadapter1.TestAdapter1
step2 - tvb.tests.framework.adapters.testadapter3.TestAdapter3
The first adapter returns a tvb.datatypes.datatype1.Datatype1 instance.
The second adapter has this passed as a dynamic workflow parameter.
We check that the steps are actually ran by checking that two operations
are created and that two dataTypes are stored.
"""
workflow_step_list = [
TestFactory.create_workflow_step(
"tvb.tests.framework.adapters.testadapter1",
"TestAdapter1",
step_index=1,
static_kwargs={
"test1_val1": 1,
"test1_val2": 1
}),
TestFactory.create_workflow_step(
"tvb.tests.framework.adapters.testadapter3",
"TestAdapter3",
step_index=2,
dynamic_kwargs={
"test": {
wf_cfg.DATATYPE_INDEX_KEY: 0,
wf_cfg.STEP_INDEX_KEY: 1
}
})
]
self.__create_complex_workflow(workflow_step_list)
stored_datatypes = dao.get_datatypes_in_project(self.test_project.id)
self.assertTrue(
len(stored_datatypes) == 3,
"DataType from all step were not stored.")
for result_row in stored_datatypes:
self.assertTrue(result_row.type in ['Datatype1', 'Datatype2'],
"Wrong type was stored.")
finished, started, error, _, _ = dao.get_operation_numbers(
self.test_project.id)
self.assertEqual(
finished, 3,
"Didn't start operations for both adapters in workflow.")
self.assertEqual(started, 0,
"Some operations from workflow didn't finish.")
self.assertEqual(error, 0,
"Some operations finished with error status.")
def test_create_workflow(self):
"""
Test that a workflow with all the associated workflow steps is actually created.
"""
workflow_step_list = [TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter2",
"TestAdapter2", step_index=1,
static_kwargs={"test2": 2}),
TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter1",
"TestAdapter1", step_index=2,
static_kwargs={"test1_val1": 1, "test1_val2": 1})]
burst_id = self.__create_complex_workflow(workflow_step_list)
workflow_entities = dao.get_workflows_for_burst(burst_id)
self.assertTrue(len(workflow_entities) == 1, "For some reason workflow was not stored in database.")
workflow_steps = dao.get_workflow_steps(workflow_entities[0].id)
self.assertEqual(len(workflow_steps), len(workflow_step_list) + 1, "Wrong number of workflow steps created.")
def test_configuration2workflow(self):
"""
Test that building a WorkflowStep from a WorkflowStepConfiguration. Make sure all the data is
correctly passed. Also check that any base_wf_step is incremented to dynamic parameters step index.
"""
workflow_step = TestFactory.create_workflow_step(
"tvb.tests.framework.adapters.testadapter1",
"TestAdapter1",
static_kwargs={"static_param": "test"},
dynamic_kwargs={
"dynamic_param": {
wf_cfg.STEP_INDEX_KEY: 0,
wf_cfg.DATATYPE_INDEX_KEY: 0
}
},
step_index=1,
base_step=5)
self.assertEqual(workflow_step.step_index, 1,
"Wrong step index in created workflow step.")
self.assertEqual(workflow_step.static_param, {'static_param': 'test'},
'Different static parameters on step.')
self.assertEqual(
workflow_step.dynamic_param, {
'dynamic_param': {
wf_cfg.STEP_INDEX_KEY: 5,
wf_cfg.DATATYPE_INDEX_KEY: 0
}
},
"Dynamic parameters not saved properly, or base workflow index not added to step index."
)
def test_workflow_generation(self):
"""
A simple test just for the fact that a workflow is created an ran,
no dynamic parameters are passed. In this case we create a two steps
workflow: step1 - tvb.tests.framework.adapters.testadapter2.TestAdapter2
step2 - tvb.tests.framework.adapters.testadapter1.TestAdapter1
The first adapter doesn't return anything and the second returns one
tvb.datatypes.datatype1.Datatype1 instance. We check that the steps
are actually ran by checking that two operations are created and that
one dataType is stored.
"""
workflow_step_list = [
TestFactory.create_workflow_step(
"tvb.tests.framework.adapters.testadapter2",
"TestAdapter2",
step_index=1,
static_kwargs={"test2": 2}),
TestFactory.create_workflow_step(
"tvb.tests.framework.adapters.testadapter1",
"TestAdapter1",
step_index=2,
static_kwargs={
"test1_val1": 1,
"test1_val2": 1
})
]
self.__create_complex_workflow(workflow_step_list)
stored_datatypes = dao.get_datatypes_in_project(self.test_project.id)
self.assertTrue(
len(stored_datatypes) == 2,
"DataType from second step was not stored.")
self.assertTrue(stored_datatypes[0].type == 'Datatype1',
"Wrong type was stored.")
self.assertTrue(stored_datatypes[1].type == 'Datatype1',
"Wrong type was stored.")
finished, started, error, _, _ = dao.get_operation_numbers(
self.test_project.id)
self.assertEqual(
finished, 3,
"Didnt start operations for both adapters in workflow.")
self.assertEqual(started, 0,
"Some operations from workflow didnt finish.")
self.assertEqual(error, 0,
"Some operations finished with error status.")
def test_configuration2workflow(self):
"""
Test that building a WorkflowStep from a WorkflowStepConfiguration. Make sure all the data is
correctly passed. Also check that any base_wf_step is incremented to dynamic parameters step index.
"""
workflow_step = TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter1", "TestAdapter1",
static_kwargs={"static_param": "test"},
dynamic_kwargs={"dynamic_param": {wf_cfg.STEP_INDEX_KEY: 0,
wf_cfg.DATATYPE_INDEX_KEY: 0}},
step_index=1, base_step=5)
self.assertEqual(workflow_step.step_index, 1, "Wrong step index in created workflow step.")
self.assertEqual(workflow_step.static_param, {'static_param': 'test'}, 'Different static parameters on step.')
self.assertEqual(workflow_step.dynamic_param, {'dynamic_param': {wf_cfg.STEP_INDEX_KEY: 5,
wf_cfg.DATATYPE_INDEX_KEY: 0}},
"Dynamic parameters not saved properly, or base workflow index not added to step index.")
def _prepare_and_launch_sync_burst(self):
"""
Private method to launch a dummy burst. Return the burst loaded after the launch finished
as well as the workflow steps that initially formed the burst.
NOTE: the burst launched by this method is a `dummy` one, meaning we do not use an actual
simulation, but instead test adapters.
"""
burst_config = TestFactory.store_burst(self.test_project.id)
workflow_step_list = []
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
stored_dt = datatypes_factory.DatatypesFactory()._store_datatype(
Datatype1())
first_step_algorithm = self.flow_service.get_algorithm_by_module_and_class(
"tvb.tests.framework.adapters.testadapter1",
"TestAdapterDatatypeInput")
metadata = {DataTypeMetaData.KEY_BURST: burst_config.id}
kwargs = {"test_dt_input": stored_dt.gid, 'test_non_dt_input': '0'}
operations, group = self.operation_service.prepare_operations(
self.test_user.id, self.test_project.id, first_step_algorithm,
first_step_algorithm.algorithm_category, metadata, **kwargs)
view_step = TestFactory.create_workflow_step(
"tvb.tests.framework.adapters.testadapter2",
"TestAdapter2", {"test2": 2}, {},
0,
0,
0,
0,
is_view_step=True)
view_step.fk_portlet = test_portlet.id
workflow_step_list.append(view_step)
workflows = self.workflow_service.create_and_store_workflow(
self.test_project.id, burst_config.id, 0, first_step_algorithm.id,
operations)
self.operation_service.prepare_operations_for_workflowsteps(
workflow_step_list, workflows, self.test_user.id, burst_config.id,
self.test_project.id, group, operations)
### Now fire the workflow and also update and store the burst configuration ##
self.operation_service.launch_operation(operations[0].id, False)
loaded_burst, _ = self.burst_service.load_burst(burst_config.id)
import_operation = dao.get_operation_by_id(stored_dt.fk_from_operation)
dao.remove_entity(import_operation.__class__, import_operation.id)
dao.remove_datatype(stored_dt.gid)
return loaded_burst, workflow_step_list
def _prepare_and_launch_sync_burst(self):
"""
Private method to launch a dummy burst. Return the burst loaded after the launch finished
as well as the workflow steps that initially formed the burst.
NOTE: the burst launched by this method is a `dummy` one, meaning we do not use an actual
simulation, but instead test adapters.
"""
burst_config = TestFactory.store_burst(self.test_project.id)
workflow_step_list = []
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
stored_dt = datatypes_factory.DatatypesFactory()._store_datatype(Datatype1())
first_step_algorithm = self.flow_service.get_algorithm_by_module_and_class(
"tvb.tests.framework.adapters.testadapter1", "TestAdapterDatatypeInput")
metadata = {DataTypeMetaData.KEY_BURST: burst_config.id}
kwargs = {"test_dt_input": stored_dt.gid, 'test_non_dt_input': '0'}
operations, group = self.operation_service.prepare_operations(self.test_user.id, self.test_project.id,
first_step_algorithm,
first_step_algorithm.algorithm_category,
metadata, **kwargs)
view_step = TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter2", "TestAdapter2",
{"test2": 2}, {}, 0, 0, 0, 0, is_view_step=True)
view_step.fk_portlet = test_portlet.id
workflow_step_list.append(view_step)
workflows = self.workflow_service.create_and_store_workflow(self.test_project.id, burst_config.id, 0,
first_step_algorithm.id, operations)
self.operation_service.prepare_operations_for_workflowsteps(workflow_step_list, workflows, self.test_user.id,
burst_config.id, self.test_project.id, group,
operations)
### Now fire the workflow and also update and store the burst configuration ##
self.operation_service.launch_operation(operations[0].id, False)
loaded_burst, _ = self.burst_service.load_burst(burst_config.id)
import_operation = dao.get_operation_by_id(stored_dt.fk_from_operation)
dao.remove_entity(import_operation.__class__, import_operation.id)
dao.remove_datatype(stored_dt.gid)
return loaded_burst, workflow_step_list
