class TestSomaWorkflow(unittest.TestCase):
def setUp(self):
default_config = SortedDictionary(("use_soma_workflow", True))
# use a custom temporary soma-workflow dir to avoid concurrent
# access problems
tmpdb = tempfile.mkstemp('', prefix='soma_workflow')
os.close(tmpdb[0])
os.unlink(tmpdb[1])
self.soma_workflow_temp_dir = tmpdb[1]
os.mkdir(self.soma_workflow_temp_dir)
swf_conf = StringIO.StringIO('[%s]\nSOMA_WORKFLOW_DIR = %s\n' \
% (socket.gethostname(), tmpdb[1]))
swconfig.Configuration.search_config_path \
= staticmethod(lambda : swf_conf)
self.study_config = StudyConfig(init_config=default_config)
self.atomic_pipeline = MyAtomicPipeline()
self.composite_pipeline = MyCompositePipeline()
def tearDown(self):
shutil.rmtree(self.soma_workflow_temp_dir)
def test_atomic_dependencies(self):
workflow = workflow_from_pipeline(self.atomic_pipeline)
dependencies = [(x.name, y.name) for x, y in workflow.dependencies]
self.assertTrue(len(dependencies) == 4)
self.assertTrue(("node1", "node2") in dependencies)
self.assertTrue(("node1", "node3") in dependencies)
self.assertTrue(("node2", "node4") in dependencies)
self.assertTrue(("node3", "node4") in dependencies)
self.assertEqual(workflow.groups, [])
def test_atomic_execution(self):
self.atomic_pipeline.workflow_ordered_nodes()
if sys.version_info >= (2, 7):
self.assertIn(
self.atomic_pipeline.workflow_repr,
('node1->node3->node2->node4', 'node1->node2->node3->node4'))
else: # python 2.6 unittest does not have assertIn()
self.assertTrue(self.atomic_pipeline.workflow_repr in \
('node1->node3->node2->node4',
'node1->node2->node3->node4'))
self.study_config.run(self.atomic_pipeline)
def test_composite_dependencies(self):
workflow = workflow_from_pipeline(self.composite_pipeline)
dependencies = [(x.name, y.name) for x, y in workflow.dependencies]
self.assertTrue(len(dependencies) == 16)
self.assertEqual(dependencies.count(("node1", "node2")), 1)
self.assertEqual(dependencies.count(("node1", "node3")), 2)
self.assertEqual(dependencies.count(("node2", "node4")), 1)
self.assertEqual(dependencies.count(("node3", "node4")), 2)
self.assertEqual(dependencies.count(("node1", "node2_input")), 1)
self.assertEqual(dependencies.count(("node2_output", "node4")), 1)
self.assertTrue(len(workflow.groups) == 1)
def test_composite_execution(self):
self.composite_pipeline.workflow_ordered_nodes()
self.assertTrue(self.composite_pipeline.workflow_repr in (
"node1->node3->node2->node4", "node1->node2->node3->node4"))
self.study_config.run(self.composite_pipeline)
class TestQCNodes(unittest.TestCase):
""" Test pipeline node types.
"""
def setUp(self):
""" Initialize the TestQCNodes class
"""
self.pipeline = MyPipeline()
self.pipeline.input = 'dummy_input'
self.pipeline.output = 'dummy_output'
self.output_directory = tempfile.mkdtemp()
self.study_config = StudyConfig(output_directory=self.output_directory)
self.pipeline.set_study_config(self.study_config)
def tearDown(self):
""" Remove temporary items.
"""
shutil.rmtree(self.output_directory)
def test_qc_active(self):
""" Method to test if the run qc option works properly.
"""
# Execute all the pipeline nodes
self.study_config.run(self.pipeline, execute_qc_nodes=True)
# Get the list of all the nodes that havec been executed
execution_list = self.pipeline.workflow_ordered_nodes()
# Go through all the executed nodes
for process_node in execution_list:
# Get the process instance that has been executed
process_instance = process_node.process
# Check that the node has been executed
self.assertEqual(process_instance.log_file, "in")
def test_qc_inactive(self):
""" Method to test if the run qc option works properly.
"""
# Execute all the pipeline nodes
self.study_config.run(self.pipeline, execute_qc_nodes=False)
# Get the list of all the nodes
execution_list = self.pipeline.workflow_ordered_nodes()
# Go through all the nodes
for process_node in execution_list:
# Get the process instance that may have been executed
process_instance = process_node.process
# Check that view nodes are not executed
if process_node.node_type == "view_node":
self.assertEqual(process_instance.log_file, None)
else:
self.assertEqual(process_instance.log_file, "in")
class TestQCNodes(unittest.TestCase):
""" Test pipeline node types.
"""
def setUp(self):
""" Initialize the TestQCNodes class
"""
self.pipeline = MyPipeline()
self.output_directory = tempfile.mkdtemp()
self.study_config = StudyConfig(output_directory=self.output_directory)
def __del__(self):
""" Remove temporary items.
"""
shutil.rmtree(self.output_directory)
def test_qc_active(self):
""" Method to test if the run qc option works properly.
"""
# Execute all the pipeline nodes
self.study_config.run(self.pipeline, executer_qc_nodes=True)
# Get the list of all the nodes that havec been executed
execution_list = self.pipeline.workflow_ordered_nodes()
# Go through all the executed nodes
for process_node in execution_list:
# Get the process instance that has been executed
process_instance = process_node.process
# Check that the node has been executed
self.assertEqual(process_instance.log_file, "in")
def test_qc_inactive(self):
""" Method to test if the run qc option works properly.
"""
# Execute all the pipeline nodes
self.study_config.run(self.pipeline, executer_qc_nodes=False)
# Get the list of all the nodes
execution_list = self.pipeline.workflow_ordered_nodes()
# Go through all the nodes
for process_node in execution_list:
# Get the process instance that may have been executed
process_instance = process_node.process
# Check that view nodes are not executed
if process_node.node_type == "view_node":
self.assertEqual(process_instance.log_file, None)
else:
self.assertEqual(process_instance.log_file, "in")
class TestSomaWorkflow(unittest.TestCase):
def setUp(self):
default_config = SortedDictionary(("use_soma_workflow", True), )
self.study_config = StudyConfig(init_config=default_config)
self.atomic_pipeline = MyAtomicPipeline()
self.composite_pipeline = MyCompositePipeline()
def tearDown(self):
swm = self.study_config.modules['SomaWorkflowConfig']
swc = swm.get_workflow_controller()
if swc is not None:
# stop workflow controller and wait for thread termination
swc.stop_engine()
def test_atomic_dependencies(self):
workflow = workflow_from_pipeline(self.atomic_pipeline)
dependencies = [(x.name, y.name) for x, y in workflow.dependencies]
self.assertTrue(len(dependencies) == 4)
self.assertTrue(("node1", "node2") in dependencies)
self.assertTrue(("node1", "node3") in dependencies)
self.assertTrue(("node2", "node4") in dependencies)
self.assertTrue(("node3", "node4") in dependencies)
self.assertEqual(workflow.groups, [])
def test_atomic_execution(self):
self.atomic_pipeline.workflow_ordered_nodes()
self.assertIn(
self.atomic_pipeline.workflow_repr,
('node1->node3->node2->node4', 'node1->node2->node3->node4'))
self.study_config.run(self.atomic_pipeline)
def test_composite_dependencies(self):
workflow = workflow_from_pipeline(self.composite_pipeline)
dependencies = [(x.name, y.name) for x, y in workflow.dependencies]
self.assertTrue(len(dependencies) == 8)
self.assertEqual(dependencies.count(("node1", "node2")), 1)
self.assertEqual(dependencies.count(("node1", "node3")), 2)
self.assertEqual(dependencies.count(("node2", "node4")), 1)
self.assertEqual(dependencies.count(("node3", "node4")), 2)
self.assertEqual(dependencies.count(("node1", "node1")), 1)
self.assertEqual(dependencies.count(("node4", "node4")), 1)
self.assertTrue(len(workflow.groups) == 1)
def test_composite_execution(self):
self.composite_pipeline.workflow_ordered_nodes()
self.assertTrue(self.composite_pipeline.workflow_repr in (
"node1->node3->node2->node4", "node1->node2->node3->node4"))
self.study_config.run(self.composite_pipeline)
class TestPipeline(unittest.TestCase):
""" Class to test a pipeline with an iterative node
"""
def setUp(self):
""" In the setup construct the pipeline and set some input parameters.
"""
self.directory = tempfile.mkdtemp(prefix="capsul_test")
self.study_config = StudyConfig()
# Construct the pipeline
self.pipeline = self.study_config.get_process_instance(MyPipeline)
# Set some input parameters
self.pipeline.input_image = [
os.path.join(self.directory, "toto"),
os.path.join(self.directory, "tutu")
]
self.pipeline.dynamic_parameter = [3, 1]
self.pipeline.other_input = 5
# build a pipeline with dependencies
self.small_pipeline \
= self.study_config.get_process_instance(MySmallPipeline)
self.small_pipeline.files_to_create = [
os.path.join(self.directory, "toto"),
os.path.join(self.directory, "tutu")
]
self.small_pipeline.dynamic_parameter = [3, 1]
self.small_pipeline.other_input = 5
# build a bigger pipeline with several levels
self.big_pipeline \
= self.study_config.get_process_instance(MyBigPipeline)
def tearDown(self):
swm = self.study_config.modules['SomaWorkflowConfig']
swc = swm.get_workflow_controller()
if swc is not None:
# stop workflow controller and wait for thread termination
swc.stop_engine()
if debug:
print('directory %s not removed.' % self.directory)
else:
shutil.rmtree(self.directory)
def test_iterative_pipeline_connection(self):
""" Test if an iterative process works correctly
"""
# create inputs
for f in self.pipeline.input_image:
with open(f, "w") as fobj:
fobj.write("input: %s\n" % f)
# Test the output connection
self.pipeline()
if sys.version_info >= (2, 7):
self.assertIn(
"toto-5.0-3.0",
[os.path.basename(f) for f in self.pipeline.output_image])
self.assertIn(
"tutu-5.0-1.0",
[os.path.basename(f) for f in self.pipeline.output_image])
else:
self.assertTrue(
"toto-5.0-3.0" in
[os.path.basename(f) for f in self.pipeline.output_image])
self.assertTrue(
"tutu-5.0-1.0" in
[os.path.basename(f) for f in self.pipeline.output_image])
self.assertEqual(
self.pipeline.other_output,
[self.pipeline.other_input, self.pipeline.other_input])
def test_iterative_pipeline_workflow(self):
self.small_pipeline.output_image = [
os.path.join(self.directory, 'toto_out'),
os.path.join(self.directory, 'tutu_out')
]
self.small_pipeline.other_output = [1., 2.]
workflow = pipeline_workflow.workflow_from_pipeline(
self.small_pipeline)
#expect 2 + 2 (iter) + 2 (barriers) jobs
self.assertEqual(len(workflow.jobs), 6)
# expect 6 dependencies:
# init -> iterative input barrier
# iterative output barrier -> end
# iterative input barrier -> iterative jobs (2)
# iterative jobs -> iterative output barrier (2)
self.assertEqual(len(workflow.dependencies), 6)
def test_iterative_big_pipeline_workflow(self):
self.big_pipeline.files_to_create = [["toto", "tutu"],
["tata", "titi", "tete"]]
self.big_pipeline.dynamic_parameter = [[1, 2], [3, 4, 5]]
self.big_pipeline.other_input = 5
self.big_pipeline.output_image = [
[
os.path.join(self.directory, 'toto_out'),
os.path.join(self.directory, 'tutu_out')
],
[
os.path.join(self.directory, 'tata_out'),
os.path.join(self.directory, 'titi_out'),
os.path.join(self.directory, 'tete_out')
]
]
self.big_pipeline.other_output = [[1.1, 2.1], [3.1, 4.1, 5.1]]
workflow = pipeline_workflow.workflow_from_pipeline(self.big_pipeline)
# expect 6 + 7 + 2 jobs
self.assertEqual(len(workflow.jobs), 15)
subjects = set()
for job in workflow.jobs:
if not job.name.startswith('DummyProcess') or '_map' in job.name \
or '_reduce' in job.name:
continue
param_dict = job.param_dict
self.assertEqual(param_dict["other_input"], 5)
subject = param_dict['input_image']
subjects.add(subject)
if sys.version_info >= (2, 7):
self.assertIn(subject,
["toto", "tutu", "tata", "titi", "tete"])
else:
self.assertTrue(
subject in ["toto", "tutu", "tata", "titi", "tete"])
self.assertEqual(subjects,
set(["toto", "tutu", "tata", "titi", "tete"]))
def test_iterative_pipeline_workflow_run(self):
import soma_workflow.constants as swconstants
import soma_workflow.client as swclient
self.small_pipeline.output_image = [
os.path.join(self.directory, 'toto_out'),
os.path.join(self.directory, 'tutu_out')
]
self.small_pipeline.other_output = [1., 2.]
workflow = pipeline_workflow.workflow_from_pipeline(
self.small_pipeline)
swclient.Helper.serialize(
os.path.join(self.directory, 'smallpipeline.workflow'), workflow)
self.study_config.use_soma_workflow = True
#controller = swclient.WorkflowController(config=config)
#try:
#wf_id = controller.submit_workflow(workflow)
print('* running pipeline...')
#swclient.Helper.wait_workflow(wf_id, controller)
self.study_config.run(self.small_pipeline)
print('* finished.')
#workflow_status = controller.workflow_status(wf_id)
#elements_status = controller.workflow_elements_status(wf_id)
#failed_jobs = [element for element in elements_status[0] \
#if element[1] != swconstants.DONE \
#or element[3][0] != swconstants.FINISHED_REGULARLY]
#if not debug:
#controller.delete_workflow(wf_id)
#self.assertTrue(workflow_status == swconstants.WORKFLOW_DONE,
#'Workflow did not finish regularly: %s' % workflow_status)
#self.assertTrue(len(failed_jobs) == 0, 'Jobs failed: %s'
#% failed_jobs)
# check output files contents
for ifname, fname in zip(self.small_pipeline.files_to_create,
self.small_pipeline.output_image):
with open(fname) as f:
content = f.read()
self.assertEqual(content, "file: %s\n" % ifname)
class TestSomaWorkflow(unittest.TestCase):
def setUp(self):
default_config = SortedDictionary(
("use_soma_workflow", True)
)
# use a custom temporary soma-workflow dir to avoid concurrent
# access problems
tmpdb = tempfile.mkstemp('', prefix='soma_workflow')
os.close(tmpdb[0])
os.unlink(tmpdb[1])
self.soma_workflow_temp_dir = tmpdb[1]
os.mkdir(self.soma_workflow_temp_dir)
swf_conf = StringIO.StringIO('[%s]\nSOMA_WORKFLOW_DIR = %s\n' \
% (socket.gethostname(), tmpdb[1]))
swconfig.Configuration.search_config_path \
= staticmethod(lambda : swf_conf)
self.study_config = StudyConfig(init_config=default_config)
self.atomic_pipeline = MyAtomicPipeline()
self.composite_pipeline = MyCompositePipeline()
def tearDown(self):
shutil.rmtree(self.soma_workflow_temp_dir)
def test_atomic_dependencies(self):
workflow = workflow_from_pipeline(self.atomic_pipeline)
dependencies = [(x.name, y.name) for x, y in workflow.dependencies]
self.assertTrue(len(dependencies) == 4)
self.assertTrue(("node1", "node2") in dependencies)
self.assertTrue(("node1", "node3") in dependencies)
self.assertTrue(("node2", "node4") in dependencies)
self.assertTrue(("node3", "node4") in dependencies)
self.assertEqual(workflow.groups, [])
def test_atomic_execution(self):
self.atomic_pipeline.workflow_ordered_nodes()
if sys.version_info >= (2, 7):
self.assertIn(self.atomic_pipeline.workflow_repr,
('node1->node3->node2->node4',
'node1->node2->node3->node4'))
else: # python 2.6 unittest does not have assertIn()
self.assertTrue(self.atomic_pipeline.workflow_repr in \
('node1->node3->node2->node4',
'node1->node2->node3->node4'))
tmp1 = tempfile.mkstemp('', prefix='capsul_swf')
os.write(tmp1[0], 'bidibidi'.encode())
os.close(tmp1[0])
tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out')
os.close(tmp2[0])
os.unlink(tmp2[1])
self.atomic_pipeline.input_image = tmp1[1]
self.atomic_pipeline.output_image = tmp2[1]
self.study_config.run(self.atomic_pipeline)
self.assertTrue(os.path.exists(tmp2[1]))
content = open(tmp2[1]).read()
self.assertEqual(content, 'bidibidibidibidi')
os.unlink(tmp1[1])
os.unlink(tmp2[1])
def test_composite_dependencies(self):
workflow = workflow_from_pipeline(self.composite_pipeline)
dependencies = [(x.name, y.name) for x, y in workflow.dependencies]
self.assertTrue(len(dependencies) == 16)
self.assertEqual(dependencies.count(("node1", "node2")), 1)
self.assertEqual(dependencies.count(("node1", "node3")), 2)
self.assertEqual(dependencies.count(("node2", "node4")), 1)
self.assertEqual(dependencies.count(("node3", "node4")), 2)
self.assertEqual(dependencies.count(("node1", "node2_input")), 1)
self.assertEqual(dependencies.count(("node2_output", "node4")), 1)
self.assertTrue(len(workflow.groups) == 1)
def test_composite_execution(self):
self.composite_pipeline.workflow_ordered_nodes()
self.assertTrue(self.composite_pipeline.workflow_repr in
("node1->node3->node2->node4",
"node1->node2->node3->node4"))
tmp1 = tempfile.mkstemp('', prefix='capsul_swf')
os.write(tmp1[0], 'bidibidi'.encode())
os.close(tmp1[0])
tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out')
os.close(tmp2[0])
os.unlink(tmp2[1])
self.composite_pipeline.input_image = tmp1[1]
self.composite_pipeline.output_image = tmp2[1]
self.study_config.run(self.composite_pipeline)
self.assertTrue(os.path.exists(tmp2[1]))
content = open(tmp2[1]).read()
self.assertEqual(content, 'bidibidibidibidibidibidi')
os.unlink(tmp1[1])
os.unlink(tmp2[1])
class TestSomaWorkflow(unittest.TestCase):
def setUp(self):
default_config = SortedDictionary(
("use_soma_workflow", True)
)
self.study_config = StudyConfig(init_config=default_config)
self.atomic_pipeline = MyAtomicPipeline()
self.composite_pipeline = MyCompositePipeline()
def tearDown(self):
swm = self.study_config.modules['SomaWorkflowConfig']
swc = swm.get_workflow_controller()
if swc is not None:
# stop workflow controller and wait for thread termination
swc.stop_engine()
def test_atomic_dependencies(self):
workflow = workflow_from_pipeline(self.atomic_pipeline)
dependencies = [(x.name, y.name) for x, y in workflow.dependencies]
self.assertTrue(len(dependencies) == 4)
self.assertTrue(("node1", "node2") in dependencies)
self.assertTrue(("node1", "node3") in dependencies)
self.assertTrue(("node2", "node4") in dependencies)
self.assertTrue(("node3", "node4") in dependencies)
self.assertEqual(workflow.groups, [])
def test_atomic_execution(self):
self.atomic_pipeline.workflow_ordered_nodes()
self.assertIn(self.atomic_pipeline.workflow_repr,
('node1->node3->node2->node4',
'node1->node2->node3->node4'))
tmp1 = tempfile.mkstemp('', prefix='capsul_swf')
os.write(tmp1[0], 'bidibidi'.encode())
os.close(tmp1[0])
tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out')
os.close(tmp2[0])
os.unlink(tmp2[1])
try:
self.atomic_pipeline.input_image = tmp1[1]
self.atomic_pipeline.output_image = tmp2[1]
self.study_config.run(self.atomic_pipeline)
self.assertTrue(os.path.exists(tmp2[1]))
with open(tmp2[1]) as f:
content = f.read()
self.assertEqual(content, 'bidibidibidibidi')
finally:
if os.path.exists(tmp1[1]):
os.unlink(tmp1[1])
if os.path.exists(tmp2[1]):
os.unlink(tmp2[1])
def test_atomic_execution_kwparams(self):
tmp1 = tempfile.mkstemp('', prefix='capsul_swf')
os.write(tmp1[0], 'bidibidi'.encode())
os.close(tmp1[0])
tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out')
os.close(tmp2[0])
os.unlink(tmp2[1])
try:
atomic_pipeline = MyAtomicPipeline()
self.study_config.run(atomic_pipeline,
input_image=tmp1[1], output_image=tmp2[1])
self.assertTrue(os.path.exists(tmp2[1]))
with open(tmp2[1]) as f:
content = f.read()
self.assertEqual(content, 'bidibidibidibidi')
finally:
if os.path.exists(tmp1[1]):
os.unlink(tmp1[1])
if os.path.exists(tmp2[1]):
os.unlink(tmp2[1])
def test_composite_dependencies(self):
workflow = workflow_from_pipeline(self.composite_pipeline)
dependencies = [(x.name, y.name) for x, y in workflow.dependencies]
self.assertEqual(len(dependencies), 8)
self.assertEqual(dependencies.count(("node1", "node2")), 1)
self.assertEqual(dependencies.count(("node1", "node3")), 2)
self.assertEqual(dependencies.count(("node2", "node4")), 1)
self.assertEqual(dependencies.count(("node3", "node4")), 2)
self.assertEqual(dependencies.count(("node1", "node1")), 1)
self.assertEqual(dependencies.count(("node4", "node4")), 1)
#self.assertEqual(dependencies.count(("node1", "node2_input")), 1)
#self.assertEqual(dependencies.count(("node2_output", "node4")), 1)
self.assertTrue(len(workflow.groups) == 1)
def test_composite_execution(self):
self.composite_pipeline.workflow_ordered_nodes()
self.assertTrue(self.composite_pipeline.workflow_repr in
("node1->node3->node2->node4",
"node1->node2->node3->node4"))
tmp1 = tempfile.mkstemp('', prefix='capsul_swf')
os.write(tmp1[0], 'bidibidi'.encode())
os.close(tmp1[0])
tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out')
os.close(tmp2[0])
os.unlink(tmp2[1])
try:
self.composite_pipeline.input_image = tmp1[1]
self.composite_pipeline.output_image = tmp2[1]
self.study_config.run(self.composite_pipeline)
self.assertTrue(os.path.exists(tmp2[1]))
with open(tmp2[1]) as f:
content = f.read()
self.assertEqual(content, 'bidibidibidibidibidibidi')
finally:
if os.path.exists(tmp1[1]):
os.unlink(tmp1[1])
if os.path.exists(tmp2[1]):
os.unlink(tmp2[1])