def test_loo_py_io(self): sc = StudyConfig() pipeline = sc.get_process_instance(PipelineLOO) py_file = tempfile.mkstemp(suffix='_capsul.py') pyfname = py_file[1] os.close(py_file[0]) self.temp_files.append(pyfname) python_export.save_py_pipeline(pipeline, pyfname) pipeline2 = sc.get_process_instance(pyfname) self._test_loo_pipeline(pipeline2)
def test_loo_xml_io(self): sc = StudyConfig() pipeline = sc.get_process_instance(PipelineLOO) xml_file = tempfile.mkstemp(suffix='_capsul.xml') xmlfname = xml_file[1] os.close(xml_file[0]) self.temp_files.append(xmlfname) xml.save_xml_pipeline(pipeline, xmlfname) pipeline2 = sc.get_process_instance(xmlfname) self._test_loo_pipeline(pipeline2)
def test_custom_nodes_py_io(self): sc = StudyConfig() pipeline = sc.get_process_instance(Pipeline1) py_file = tempfile.mkstemp(suffix='_capsul.py') pyfname = py_file[1] os.close(py_file[0]) self.add_py_tmpfile(pyfname) python_export.save_py_pipeline(pipeline, pyfname) pipeline2 = sc.get_process_instance(pyfname) self._test_custom_nodes(pipeline)
def test_custom_nodes_workflow(self): sc = StudyConfig() pipeline = sc.get_process_instance(Pipeline1) pipeline.main_input = '/dir/file' pipeline.output_directory = '/dir/out_dir' wf = pipeline_workflow.workflow_from_pipeline(pipeline, create_directories=False) self.assertEqual(len(wf.jobs), 3) self.assertEqual(len(wf.dependencies), 2) self.assertEqual( sorted([[x.name for x in d] for d in wf.dependencies]), sorted([['train1', 'train2'], ['train2', 'test']]))
def test_custom_nodes_workflow(self): sc = StudyConfig() pipeline = sc.get_process_instance(Pipeline1) pipeline.main_input = os.path.join(self.temp_dir, 'file') pipeline.output_directory = os.path.join(self.temp_dir, 'out_dir') wf = pipeline_workflow.workflow_from_pipeline(pipeline, create_directories=False) self.assertEqual(len(wf.jobs), 7) self.assertEqual(len(wf.dependencies), 6) self.assertEqual( sorted([[x.name for x in d] for d in wf.dependencies]), sorted([['LOO', 'train1'], ['train1', 'train2'], ['train1', 'intermediate_output'], ['train2', 'test'], ['train2', 'output_file'], ['test', 'test_output']]))
def test_mapreduce(self): sc = StudyConfig() pipeline = sc.get_process_instance(PipelineMapReduce) pipeline.main_inputs = [ os.path.join(self.temp_dir, 'file%d' % i) for i in range(4) ] pipeline.subjects = ['Robert', 'Gustave'] pipeline.output_directory = os.path.join(self.temp_dir, 'out_dir') self.assertEqual(pipeline.nodes['cat'].process.files, [ os.path.join(pipeline.output_directory, '%s_test_output' % pipeline.subjects[0]), os.path.join(pipeline.output_directory, '%s_test_output' % pipeline.subjects[1]) ]) wf = pipeline_workflow.workflow_from_pipeline(pipeline, create_directories=False) self.assertEqual(len(wf.jobs), 19) #print(sorted([(d[0].name, d[1].name) for d in wf.dependencies])) self.assertEqual(len(wf.dependencies), 28)
class TestPipeline(unittest.TestCase): """ Class to test a pipeline with an iterative node """ def setUp(self): """ In the setup construct the pipeline and set some input parameters. """ self.directory = tempfile.mkdtemp(prefix="capsul_test") self.study_config = StudyConfig() # Construct the pipeline self.pipeline = self.study_config.get_process_instance(MyPipeline) # Set some input parameters self.pipeline.input_image = [ os.path.join(self.directory, "toto"), os.path.join(self.directory, "tutu") ] self.pipeline.dynamic_parameter = [3, 1] self.pipeline.other_input = 5 # build a pipeline with dependencies self.small_pipeline \ = self.study_config.get_process_instance(MySmallPipeline) self.small_pipeline.files_to_create = [ os.path.join(self.directory, "toto"), os.path.join(self.directory, "tutu") ] self.small_pipeline.dynamic_parameter = [3, 1] self.small_pipeline.other_input = 5 # build a bigger pipeline with several levels self.big_pipeline \ = self.study_config.get_process_instance(MyBigPipeline) def tearDown(self): swm = self.study_config.modules['SomaWorkflowConfig'] swc = swm.get_workflow_controller() if swc is not None: # stop workflow controller and wait for thread termination swc.stop_engine() if debug: print('directory %s not removed.' % self.directory) else: shutil.rmtree(self.directory) def test_iterative_pipeline_connection(self): """ Test if an iterative process works correctly """ # create inputs for f in self.pipeline.input_image: with open(f, "w") as fobj: fobj.write("input: %s\n" % f) # Test the output connection self.pipeline() if sys.version_info >= (2, 7): self.assertIn( "toto-5.0-3.0", [os.path.basename(f) for f in self.pipeline.output_image]) self.assertIn( "tutu-5.0-1.0", [os.path.basename(f) for f in self.pipeline.output_image]) else: self.assertTrue( "toto-5.0-3.0" in [os.path.basename(f) for f in self.pipeline.output_image]) self.assertTrue( "tutu-5.0-1.0" in [os.path.basename(f) for f in self.pipeline.output_image]) self.assertEqual( self.pipeline.other_output, [self.pipeline.other_input, self.pipeline.other_input]) def test_iterative_pipeline_workflow(self): self.small_pipeline.output_image = [ os.path.join(self.directory, 'toto_out'), os.path.join(self.directory, 'tutu_out') ] self.small_pipeline.other_output = [1., 2.] workflow = pipeline_workflow.workflow_from_pipeline( self.small_pipeline) #expect 2 + 2 (iter) + 2 (barriers) jobs self.assertEqual(len(workflow.jobs), 6) # expect 6 dependencies: # init -> iterative input barrier # iterative output barrier -> end # iterative input barrier -> iterative jobs (2) # iterative jobs -> iterative output barrier (2) self.assertEqual(len(workflow.dependencies), 6) def test_iterative_big_pipeline_workflow(self): self.big_pipeline.files_to_create = [["toto", "tutu"], ["tata", "titi", "tete"]] self.big_pipeline.dynamic_parameter = [[1, 2], [3, 4, 5]] self.big_pipeline.other_input = 5 self.big_pipeline.output_image = [ [ os.path.join(self.directory, 'toto_out'), os.path.join(self.directory, 'tutu_out') ], [ os.path.join(self.directory, 'tata_out'), os.path.join(self.directory, 'titi_out'), os.path.join(self.directory, 'tete_out') ] ] self.big_pipeline.other_output = [[1.1, 2.1], [3.1, 4.1, 5.1]] workflow = pipeline_workflow.workflow_from_pipeline(self.big_pipeline) # expect 6 + 7 + 2 jobs self.assertEqual(len(workflow.jobs), 15) subjects = set() for job in workflow.jobs: if not job.name.startswith('DummyProcess') or '_map' in job.name \ or '_reduce' in job.name: continue param_dict = job.param_dict self.assertEqual(param_dict["other_input"], 5) subject = param_dict['input_image'] subjects.add(subject) if sys.version_info >= (2, 7): self.assertIn(subject, ["toto", "tutu", "tata", "titi", "tete"]) else: self.assertTrue( subject in ["toto", "tutu", "tata", "titi", "tete"]) self.assertEqual(subjects, set(["toto", "tutu", "tata", "titi", "tete"])) def test_iterative_pipeline_workflow_run(self): import soma_workflow.constants as swconstants import soma_workflow.client as swclient self.small_pipeline.output_image = [ os.path.join(self.directory, 'toto_out'), os.path.join(self.directory, 'tutu_out') ] self.small_pipeline.other_output = [1., 2.] workflow = pipeline_workflow.workflow_from_pipeline( self.small_pipeline) swclient.Helper.serialize( os.path.join(self.directory, 'smallpipeline.workflow'), workflow) self.study_config.use_soma_workflow = True #controller = swclient.WorkflowController(config=config) #try: #wf_id = controller.submit_workflow(workflow) print('* running pipeline...') #swclient.Helper.wait_workflow(wf_id, controller) self.study_config.run(self.small_pipeline) print('* finished.') #workflow_status = controller.workflow_status(wf_id) #elements_status = controller.workflow_elements_status(wf_id) #failed_jobs = [element for element in elements_status[0] \ #if element[1] != swconstants.DONE \ #or element[3][0] != swconstants.FINISHED_REGULARLY] #if not debug: #controller.delete_workflow(wf_id) #self.assertTrue(workflow_status == swconstants.WORKFLOW_DONE, #'Workflow did not finish regularly: %s' % workflow_status) #self.assertTrue(len(failed_jobs) == 0, 'Jobs failed: %s' #% failed_jobs) # check output files contents for ifname, fname in zip(self.small_pipeline.files_to_create, self.small_pipeline.output_image): with open(fname) as f: content = f.read() self.assertEqual(content, "file: %s\n" % ifname)
def test_leave_one_out_pipeline(self): sc = StudyConfig() pipeline = sc.get_process_instance(PipelineLOO) self._test_loo_pipeline(pipeline)
def test_custom_nodes(self): sc = StudyConfig() pipeline = sc.get_process_instance(Pipeline1) self._test_custom_nodes(pipeline)
# this docwriter is juste used to manage short names docwriter = PipelineHelpWriter([], short_names=short_names) # Where the documentation will be written: a relative path from the # makefile short_name = docwriter.get_short_name(module_name) outdir = os.path.join(base_outdir, short_name, "schema") if not os.path.isdir(outdir): os.makedirs(outdir) # Go through all pipeline for module_pipeline in module_pipelines: # Get pipeline instance pipeline_instance = study_config.get_process_instance(module_pipeline) # Get output files short_pipeline = docwriter.get_short_name(module_pipeline) image_name = os.path.join(outdir, short_pipeline + ".png") pipeline_tools.save_dot_image( pipeline_instance, image_name, nodesep=0.1, include_io=False, rankdir='TB') logger.info("Pipeline '{0}' representation has been written at " "location '{1}'.".format(module_pipeline, os.path.abspath(image_name))) # Just print a summary logger.info("Summary: '{0}' files written for module '{1}'.".format( len(module_pipelines), module_name))