def test_loo_py_io(self):
sc = StudyConfig()
pipeline = sc.get_process_instance(PipelineLOO)
py_file = tempfile.mkstemp(suffix='_capsul.py')
pyfname = py_file[1]
os.close(py_file[0])
self.temp_files.append(pyfname)
python_export.save_py_pipeline(pipeline, pyfname)
pipeline2 = sc.get_process_instance(pyfname)
self._test_loo_pipeline(pipeline2)
def test_loo_xml_io(self):
sc = StudyConfig()
pipeline = sc.get_process_instance(PipelineLOO)
xml_file = tempfile.mkstemp(suffix='_capsul.xml')
xmlfname = xml_file[1]
os.close(xml_file[0])
self.temp_files.append(xmlfname)
xml.save_xml_pipeline(pipeline, xmlfname)
pipeline2 = sc.get_process_instance(xmlfname)
self._test_loo_pipeline(pipeline2)
def test_custom_nodes_py_io(self):
sc = StudyConfig()
pipeline = sc.get_process_instance(Pipeline1)
py_file = tempfile.mkstemp(suffix='_capsul.py')
pyfname = py_file[1]
os.close(py_file[0])
self.add_py_tmpfile(pyfname)
python_export.save_py_pipeline(pipeline, pyfname)
pipeline2 = sc.get_process_instance(pyfname)
self._test_custom_nodes(pipeline)
def test_custom_nodes_workflow(self):
sc = StudyConfig()
pipeline = sc.get_process_instance(Pipeline1)
pipeline.main_input = '/dir/file'
pipeline.output_directory = '/dir/out_dir'
wf = pipeline_workflow.workflow_from_pipeline(pipeline,
create_directories=False)
self.assertEqual(len(wf.jobs), 3)
self.assertEqual(len(wf.dependencies), 2)
self.assertEqual(
sorted([[x.name for x in d] for d in wf.dependencies]),
sorted([['train1', 'train2'], ['train2', 'test']]))
def test_custom_nodes_workflow(self):
sc = StudyConfig()
pipeline = sc.get_process_instance(Pipeline1)
pipeline.main_input = os.path.join(self.temp_dir, 'file')
pipeline.output_directory = os.path.join(self.temp_dir, 'out_dir')
wf = pipeline_workflow.workflow_from_pipeline(pipeline,
create_directories=False)
self.assertEqual(len(wf.jobs), 7)
self.assertEqual(len(wf.dependencies), 6)
self.assertEqual(
sorted([[x.name for x in d] for d in wf.dependencies]),
sorted([['LOO', 'train1'], ['train1', 'train2'],
['train1', 'intermediate_output'], ['train2', 'test'],
['train2', 'output_file'], ['test', 'test_output']]))
def test_mapreduce(self):
sc = StudyConfig()
pipeline = sc.get_process_instance(PipelineMapReduce)
pipeline.main_inputs = [
os.path.join(self.temp_dir, 'file%d' % i) for i in range(4)
]
pipeline.subjects = ['Robert', 'Gustave']
pipeline.output_directory = os.path.join(self.temp_dir, 'out_dir')
self.assertEqual(pipeline.nodes['cat'].process.files, [
os.path.join(pipeline.output_directory,
'%s_test_output' % pipeline.subjects[0]),
os.path.join(pipeline.output_directory,
'%s_test_output' % pipeline.subjects[1])
])
wf = pipeline_workflow.workflow_from_pipeline(pipeline,
create_directories=False)
self.assertEqual(len(wf.jobs), 19)
#print(sorted([(d[0].name, d[1].name) for d in wf.dependencies]))
self.assertEqual(len(wf.dependencies), 28)
class TestPipeline(unittest.TestCase):
""" Class to test a pipeline with an iterative node
"""
def setUp(self):
""" In the setup construct the pipeline and set some input parameters.
"""
self.directory = tempfile.mkdtemp(prefix="capsul_test")
self.study_config = StudyConfig()
# Construct the pipeline
self.pipeline = self.study_config.get_process_instance(MyPipeline)
# Set some input parameters
self.pipeline.input_image = [
os.path.join(self.directory, "toto"),
os.path.join(self.directory, "tutu")
]
self.pipeline.dynamic_parameter = [3, 1]
self.pipeline.other_input = 5
# build a pipeline with dependencies
self.small_pipeline \
= self.study_config.get_process_instance(MySmallPipeline)
self.small_pipeline.files_to_create = [
os.path.join(self.directory, "toto"),
os.path.join(self.directory, "tutu")
]
self.small_pipeline.dynamic_parameter = [3, 1]
self.small_pipeline.other_input = 5
# build a bigger pipeline with several levels
self.big_pipeline \
= self.study_config.get_process_instance(MyBigPipeline)
def tearDown(self):
swm = self.study_config.modules['SomaWorkflowConfig']
swc = swm.get_workflow_controller()
if swc is not None:
# stop workflow controller and wait for thread termination
swc.stop_engine()
if debug:
print('directory %s not removed.' % self.directory)
else:
shutil.rmtree(self.directory)
def test_iterative_pipeline_connection(self):
""" Test if an iterative process works correctly
"""
# create inputs
for f in self.pipeline.input_image:
with open(f, "w") as fobj:
fobj.write("input: %s\n" % f)
# Test the output connection
self.pipeline()
if sys.version_info >= (2, 7):
self.assertIn(
"toto-5.0-3.0",
[os.path.basename(f) for f in self.pipeline.output_image])
self.assertIn(
"tutu-5.0-1.0",
[os.path.basename(f) for f in self.pipeline.output_image])
else:
self.assertTrue(
"toto-5.0-3.0" in
[os.path.basename(f) for f in self.pipeline.output_image])
self.assertTrue(
"tutu-5.0-1.0" in
[os.path.basename(f) for f in self.pipeline.output_image])
self.assertEqual(
self.pipeline.other_output,
[self.pipeline.other_input, self.pipeline.other_input])
def test_iterative_pipeline_workflow(self):
self.small_pipeline.output_image = [
os.path.join(self.directory, 'toto_out'),
os.path.join(self.directory, 'tutu_out')
]
self.small_pipeline.other_output = [1., 2.]
workflow = pipeline_workflow.workflow_from_pipeline(
self.small_pipeline)
#expect 2 + 2 (iter) + 2 (barriers) jobs
self.assertEqual(len(workflow.jobs), 6)
# expect 6 dependencies:
# init -> iterative input barrier
# iterative output barrier -> end
# iterative input barrier -> iterative jobs (2)
# iterative jobs -> iterative output barrier (2)
self.assertEqual(len(workflow.dependencies), 6)
def test_iterative_big_pipeline_workflow(self):
self.big_pipeline.files_to_create = [["toto", "tutu"],
["tata", "titi", "tete"]]
self.big_pipeline.dynamic_parameter = [[1, 2], [3, 4, 5]]
self.big_pipeline.other_input = 5
self.big_pipeline.output_image = [
[
os.path.join(self.directory, 'toto_out'),
os.path.join(self.directory, 'tutu_out')
],
[
os.path.join(self.directory, 'tata_out'),
os.path.join(self.directory, 'titi_out'),
os.path.join(self.directory, 'tete_out')
]
]
self.big_pipeline.other_output = [[1.1, 2.1], [3.1, 4.1, 5.1]]
workflow = pipeline_workflow.workflow_from_pipeline(self.big_pipeline)
# expect 6 + 7 + 2 jobs
self.assertEqual(len(workflow.jobs), 15)
subjects = set()
for job in workflow.jobs:
if not job.name.startswith('DummyProcess') or '_map' in job.name \
or '_reduce' in job.name:
continue
param_dict = job.param_dict
self.assertEqual(param_dict["other_input"], 5)
subject = param_dict['input_image']
subjects.add(subject)
if sys.version_info >= (2, 7):
self.assertIn(subject,
["toto", "tutu", "tata", "titi", "tete"])
else:
self.assertTrue(
subject in ["toto", "tutu", "tata", "titi", "tete"])
self.assertEqual(subjects,
set(["toto", "tutu", "tata", "titi", "tete"]))
def test_iterative_pipeline_workflow_run(self):
import soma_workflow.constants as swconstants
import soma_workflow.client as swclient
self.small_pipeline.output_image = [
os.path.join(self.directory, 'toto_out'),
os.path.join(self.directory, 'tutu_out')
]
self.small_pipeline.other_output = [1., 2.]
workflow = pipeline_workflow.workflow_from_pipeline(
self.small_pipeline)
swclient.Helper.serialize(
os.path.join(self.directory, 'smallpipeline.workflow'), workflow)
self.study_config.use_soma_workflow = True
#controller = swclient.WorkflowController(config=config)
#try:
#wf_id = controller.submit_workflow(workflow)
print('* running pipeline...')
#swclient.Helper.wait_workflow(wf_id, controller)
self.study_config.run(self.small_pipeline)
print('* finished.')
#workflow_status = controller.workflow_status(wf_id)
#elements_status = controller.workflow_elements_status(wf_id)
#failed_jobs = [element for element in elements_status[0] \
#if element[1] != swconstants.DONE \
#or element[3][0] != swconstants.FINISHED_REGULARLY]
#if not debug:
#controller.delete_workflow(wf_id)
#self.assertTrue(workflow_status == swconstants.WORKFLOW_DONE,
#'Workflow did not finish regularly: %s' % workflow_status)
#self.assertTrue(len(failed_jobs) == 0, 'Jobs failed: %s'
#% failed_jobs)
# check output files contents
for ifname, fname in zip(self.small_pipeline.files_to_create,
self.small_pipeline.output_image):
with open(fname) as f:
content = f.read()
self.assertEqual(content, "file: %s\n" % ifname)
def test_leave_one_out_pipeline(self):
sc = StudyConfig()
pipeline = sc.get_process_instance(PipelineLOO)
self._test_loo_pipeline(pipeline)
def test_custom_nodes(self):
sc = StudyConfig()
pipeline = sc.get_process_instance(Pipeline1)
self._test_custom_nodes(pipeline)
# this docwriter is juste used to manage short names
docwriter = PipelineHelpWriter([], short_names=short_names)
# Where the documentation will be written: a relative path from the
# makefile
short_name = docwriter.get_short_name(module_name)
outdir = os.path.join(base_outdir, short_name, "schema")
if not os.path.isdir(outdir):
os.makedirs(outdir)
# Go through all pipeline
for module_pipeline in module_pipelines:
# Get pipeline instance
pipeline_instance = study_config.get_process_instance(module_pipeline)
# Get output files
short_pipeline = docwriter.get_short_name(module_pipeline)
image_name = os.path.join(outdir, short_pipeline + ".png")
pipeline_tools.save_dot_image(
pipeline_instance, image_name, nodesep=0.1, include_io=False,
rankdir='TB')
logger.info("Pipeline '{0}' representation has been written at "
"location '{1}'.".format(module_pipeline,
os.path.abspath(image_name)))
# Just print a summary
logger.info("Summary: '{0}' files written for module '{1}'.".format(
len(module_pipelines), module_name))
