def setUp(self): """ In the setup construct the pipeline and set some input parameters. """ self.directory = tempfile.mkdtemp(prefix="capsul_test") self.study_config = StudyConfig() # Construct the pipeline self.pipeline = self.study_config.get_process_instance(MyPipeline) # Set some input parameters self.pipeline.input_image = [ os.path.join(self.directory, "toto"), os.path.join(self.directory, "tutu") ] self.pipeline.dynamic_parameter = [3, 1] self.pipeline.other_input = 5 # build a pipeline with dependencies self.small_pipeline \ = self.study_config.get_process_instance(MySmallPipeline) self.small_pipeline.files_to_create = [ os.path.join(self.directory, "toto"), os.path.join(self.directory, "tutu") ] self.small_pipeline.dynamic_parameter = [3, 1] self.small_pipeline.other_input = 5 # build a bigger pipeline with several levels self.big_pipeline \ = self.study_config.get_process_instance(MyBigPipeline)
class TestSomaWorkflow(unittest.TestCase): def setUp(self): default_config = SortedDictionary(("use_soma_workflow", True)) # use a custom temporary soma-workflow dir to avoid concurrent # access problems tmpdb = tempfile.mkstemp('', prefix='soma_workflow') os.close(tmpdb[0]) os.unlink(tmpdb[1]) self.soma_workflow_temp_dir = tmpdb[1] os.mkdir(self.soma_workflow_temp_dir) swf_conf = StringIO.StringIO('[%s]\nSOMA_WORKFLOW_DIR = %s\n' \ % (socket.gethostname(), tmpdb[1])) swconfig.Configuration.search_config_path \ = staticmethod(lambda : swf_conf) self.study_config = StudyConfig(init_config=default_config) self.atomic_pipeline = MyAtomicPipeline() self.composite_pipeline = MyCompositePipeline() def tearDown(self): shutil.rmtree(self.soma_workflow_temp_dir) def test_atomic_dependencies(self): workflow = workflow_from_pipeline(self.atomic_pipeline) dependencies = [(x.name, y.name) for x, y in workflow.dependencies] self.assertTrue(len(dependencies) == 4) self.assertTrue(("node1", "node2") in dependencies) self.assertTrue(("node1", "node3") in dependencies) self.assertTrue(("node2", "node4") in dependencies) self.assertTrue(("node3", "node4") in dependencies) self.assertEqual(workflow.groups, []) def test_atomic_execution(self): self.atomic_pipeline.workflow_ordered_nodes() if sys.version_info >= (2, 7): self.assertIn( self.atomic_pipeline.workflow_repr, ('node1->node3->node2->node4', 'node1->node2->node3->node4')) else: # python 2.6 unittest does not have assertIn() self.assertTrue(self.atomic_pipeline.workflow_repr in \ ('node1->node3->node2->node4', 'node1->node2->node3->node4')) self.study_config.run(self.atomic_pipeline) def test_composite_dependencies(self): workflow = workflow_from_pipeline(self.composite_pipeline) dependencies = [(x.name, y.name) for x, y in workflow.dependencies] self.assertTrue(len(dependencies) == 16) self.assertEqual(dependencies.count(("node1", "node2")), 1) self.assertEqual(dependencies.count(("node1", "node3")), 2) self.assertEqual(dependencies.count(("node2", "node4")), 1) self.assertEqual(dependencies.count(("node3", "node4")), 2) self.assertEqual(dependencies.count(("node1", "node2_input")), 1) self.assertEqual(dependencies.count(("node2_output", "node4")), 1) self.assertTrue(len(workflow.groups) == 1) def test_composite_execution(self): self.composite_pipeline.workflow_ordered_nodes() self.assertTrue(self.composite_pipeline.workflow_repr in ( "node1->node3->node2->node4", "node1->node2->node3->node4")) self.study_config.run(self.composite_pipeline)
def setUp(self): default_config = SortedDictionary( ("use_soma_workflow", True) ) self.study_config = StudyConfig(init_config=default_config) self.atomic_pipeline = MyAtomicPipeline() self.composite_pipeline = MyCompositePipeline()
def setUp(self): """ Initialize the TestQCNodes class """ self.pipeline = MyPipeline() self.pipeline.input = 'dummy_input' self.pipeline.output = 'dummy_output' self.output_directory = tempfile.mkdtemp() self.study_config = StudyConfig(output_directory=self.output_directory)
class TestQCNodes(unittest.TestCase): """ Test pipeline node types. """ def setUp(self): """ Initialize the TestQCNodes class """ self.pipeline = MyPipeline() self.pipeline.input = 'dummy_input' self.pipeline.output = 'dummy_output' self.output_directory = tempfile.mkdtemp() self.study_config = StudyConfig(output_directory=self.output_directory) self.pipeline.set_study_config(self.study_config) def tearDown(self): """ Remove temporary items. """ shutil.rmtree(self.output_directory) def test_qc_active(self): """ Method to test if the run qc option works properly. """ # Execute all the pipeline nodes self.study_config.run(self.pipeline, execute_qc_nodes=True) # Get the list of all the nodes that havec been executed execution_list = self.pipeline.workflow_ordered_nodes() # Go through all the executed nodes for process_node in execution_list: # Get the process instance that has been executed process_instance = process_node.process # Check that the node has been executed self.assertEqual(process_instance.log_file, "in") def test_qc_inactive(self): """ Method to test if the run qc option works properly. """ # Execute all the pipeline nodes self.study_config.run(self.pipeline, execute_qc_nodes=False) # Get the list of all the nodes execution_list = self.pipeline.workflow_ordered_nodes() # Go through all the nodes for process_node in execution_list: # Get the process instance that may have been executed process_instance = process_node.process # Check that view nodes are not executed if process_node.node_type == "view_node": self.assertEqual(process_instance.log_file, None) else: self.assertEqual(process_instance.log_file, "in")
def test_loo_py_io(self): sc = StudyConfig() pipeline = sc.get_process_instance(PipelineLOO) py_file = tempfile.mkstemp(suffix='_capsul.py') pyfname = py_file[1] os.close(py_file[0]) self.temp_files.append(pyfname) python_export.save_py_pipeline(pipeline, pyfname) pipeline2 = sc.get_process_instance(pyfname) self._test_loo_pipeline(pipeline2)
def test_custom_nodes_py_io(self): sc = StudyConfig() pipeline = sc.get_process_instance(Pipeline1) py_file = tempfile.mkstemp(suffix='_capsul.py') pyfname = py_file[1] os.close(py_file[0]) self.add_py_tmpfile(pyfname) python_export.save_py_pipeline(pipeline, pyfname) pipeline2 = sc.get_process_instance(pyfname) self._test_custom_nodes(pipeline)
def test_loo_xml_io(self): sc = StudyConfig() pipeline = sc.get_process_instance(PipelineLOO) xml_file = tempfile.mkstemp(suffix='_capsul.xml') xmlfname = xml_file[1] os.close(xml_file[0]) self.temp_files.append(xmlfname) xml.save_xml_pipeline(pipeline, xmlfname) pipeline2 = sc.get_process_instance(xmlfname) self._test_loo_pipeline(pipeline2)
def init_study_config(init_config={}): study_config = StudyConfig('test_study', modules=['FomConfig', 'SomaWorkflowConfig'], init_config=init_config) study_config.input_directory = '/tmp/in' study_config.output_directory = '/tmp/out' study_config.attributes_schema_paths.append( 'capsul.attributes.test.test_attributed_process') return study_config
class TestQCNodes(unittest.TestCase): """ Test pipeline node types. """ def setUp(self): """ Initialize the TestQCNodes class """ self.pipeline = MyPipeline() self.output_directory = tempfile.mkdtemp() self.study_config = StudyConfig(output_directory=self.output_directory) def __del__(self): """ Remove temporary items. """ shutil.rmtree(self.output_directory) def test_qc_active(self): """ Method to test if the run qc option works properly. """ # Execute all the pipeline nodes self.study_config.run(self.pipeline, executer_qc_nodes=True) # Get the list of all the nodes that havec been executed execution_list = self.pipeline.workflow_ordered_nodes() # Go through all the executed nodes for process_node in execution_list: # Get the process instance that has been executed process_instance = process_node.process # Check that the node has been executed self.assertEqual(process_instance.log_file, "in") def test_qc_inactive(self): """ Method to test if the run qc option works properly. """ # Execute all the pipeline nodes self.study_config.run(self.pipeline, executer_qc_nodes=False) # Get the list of all the nodes execution_list = self.pipeline.workflow_ordered_nodes() # Go through all the nodes for process_node in execution_list: # Get the process instance that may have been executed process_instance = process_node.process # Check that view nodes are not executed if process_node.node_type == "view_node": self.assertEqual(process_instance.log_file, None) else: self.assertEqual(process_instance.log_file, "in")
def test_custom_nodes_workflow(self): sc = StudyConfig() pipeline = sc.get_process_instance(Pipeline1) pipeline.main_input = '/dir/file' pipeline.output_directory = '/dir/out_dir' wf = pipeline_workflow.workflow_from_pipeline(pipeline, create_directories=False) self.assertEqual(len(wf.jobs), 3) self.assertEqual(len(wf.dependencies), 2) self.assertEqual( sorted([[x.name for x in d] for d in wf.dependencies]), sorted([['train1', 'train2'], ['train2', 'test']]))
def test_custom_nodes_workflow(self): sc = StudyConfig() pipeline = sc.get_process_instance(Pipeline1) pipeline.main_input = os.path.join(self.temp_dir, 'file') pipeline.output_directory = os.path.join(self.temp_dir, 'out_dir') wf = pipeline_workflow.workflow_from_pipeline(pipeline, create_directories=False) self.assertEqual(len(wf.jobs), 7) self.assertEqual(len(wf.dependencies), 6) self.assertEqual( sorted([[x.name for x in d] for d in wf.dependencies]), sorted([['LOO', 'train1'], ['train1', 'train2'], ['train1', 'intermediate_output'], ['train2', 'test'], ['train2', 'output_file'], ['test', 'test_output']]))
class TestSomaWorkflow(unittest.TestCase): def setUp(self): default_config = SortedDictionary(("use_soma_workflow", True), ) self.study_config = StudyConfig(init_config=default_config) self.atomic_pipeline = MyAtomicPipeline() self.composite_pipeline = MyCompositePipeline() def tearDown(self): swm = self.study_config.modules['SomaWorkflowConfig'] swc = swm.get_workflow_controller() if swc is not None: # stop workflow controller and wait for thread termination swc.stop_engine() def test_atomic_dependencies(self): workflow = workflow_from_pipeline(self.atomic_pipeline) dependencies = [(x.name, y.name) for x, y in workflow.dependencies] self.assertTrue(len(dependencies) == 4) self.assertTrue(("node1", "node2") in dependencies) self.assertTrue(("node1", "node3") in dependencies) self.assertTrue(("node2", "node4") in dependencies) self.assertTrue(("node3", "node4") in dependencies) self.assertEqual(workflow.groups, []) def test_atomic_execution(self): self.atomic_pipeline.workflow_ordered_nodes() self.assertIn( self.atomic_pipeline.workflow_repr, ('node1->node3->node2->node4', 'node1->node2->node3->node4')) self.study_config.run(self.atomic_pipeline) def test_composite_dependencies(self): workflow = workflow_from_pipeline(self.composite_pipeline) dependencies = [(x.name, y.name) for x, y in workflow.dependencies] self.assertTrue(len(dependencies) == 8) self.assertEqual(dependencies.count(("node1", "node2")), 1) self.assertEqual(dependencies.count(("node1", "node3")), 2) self.assertEqual(dependencies.count(("node2", "node4")), 1) self.assertEqual(dependencies.count(("node3", "node4")), 2) self.assertEqual(dependencies.count(("node1", "node1")), 1) self.assertEqual(dependencies.count(("node4", "node4")), 1) self.assertTrue(len(workflow.groups) == 1) def test_composite_execution(self): self.composite_pipeline.workflow_ordered_nodes() self.assertTrue(self.composite_pipeline.workflow_repr in ( "node1->node3->node2->node4", "node1->node2->node3->node4")) self.study_config.run(self.composite_pipeline)
def display_parameters(self, item): """ This method was used to display the parameters of a process. It will be useful to generate processes in the near future. """ study_config = StudyConfig(modules=StudyConfig.default_modules + ['NipypeConfig']) process_name = item.text(0) list_path = [] while item is not self.process_library.topLevelItem(0): item = item.parent() list_path.append(item.text(0)) list_path = list(reversed(list_path)) package_name = '.'.join(list_path) __import__(package_name) pkg = sys.modules[package_name] for k, v in sorted(list(pkg.__dict__.items())): if k == process_name: try: process = get_process_instance(v) except: print('AIEEEE') pass else: print(process.get_inputs()) txt = "Inputs: \n" + str(v.input_spec()) txt2 = "\nOutputs: \n" + str(v.output_spec()) self.label_test.setText(txt + txt2)
def setUp(self): default_config = SortedDictionary(("use_soma_workflow", True)) # use a custom temporary soma-workflow dir to avoid concurrent # access problems tmpdb = tempfile.mkstemp('', prefix='soma_workflow') os.close(tmpdb[0]) os.unlink(tmpdb[1]) self.soma_workflow_temp_dir = tmpdb[1] os.mkdir(self.soma_workflow_temp_dir) swf_conf = '[%s]\nSOMA_WORKFLOW_DIR = %s\n' \ % (socket.gethostname(), tmpdb[1]) swconfig.Configuration.search_config_path \ = staticmethod(lambda : StringIO.StringIO(swf_conf)) self.study_config = StudyConfig(init_config=default_config) self.atomic_pipeline = MyAtomicPipeline() self.composite_pipeline = MyCompositePipeline()
def setUp(self): tmpout = tempfile.mkdtemp(prefix='capsul_ex_test_') self.work_dir = tmpout print('working dir:', tmpout) self.input = os.path.join(self.work_dir, 'input_data') stdout = open(os.path.join(self.work_dir, 'stdout'), 'w') subprocess.check_call(['generate_data', self.input], stdout=stdout, stderr=stdout) del stdout self.output = os.path.join(self.work_dir, 'output_data') study_config = StudyConfig(modules=['SomaWorkflowConfig', 'AttributesConfig']) study_config.input_directory = self.input study_config.output_directory = os.path.join( self.work_dir, 'output_data') study_config.shared_directory = os.path.join(self.input, 'share') study_config.somaworkflow_computing_resource = 'localhost' study_config.somaworkflow_computing_resources_config.localhost = { 'transfer_paths': [], } study_config.attributes_schema_paths += [ 'bv_capsul_ex.adhoc_completion', 'bv_capsul_ex.tests.test_ex_pipeline', 'bv_capsul_ex.schema'] study_config.process_completion = 'bv_capsul_ex' study_config.attributes_schemas = {'input': 'bv_capsul_ex', 'output': 'bv_capsul_ex', 'shared': 'bv_capsul_shared'} self.study_config = study_config
def test_mapreduce(self): sc = StudyConfig() pipeline = sc.get_process_instance(PipelineMapReduce) pipeline.main_inputs = [ os.path.join(self.temp_dir, 'file%d' % i) for i in range(4) ] pipeline.subjects = ['Robert', 'Gustave'] pipeline.output_directory = os.path.join(self.temp_dir, 'out_dir') self.assertEqual(pipeline.nodes['cat'].process.files, [ os.path.join(pipeline.output_directory, '%s_test_output' % pipeline.subjects[0]), os.path.join(pipeline.output_directory, '%s_test_output' % pipeline.subjects[1]) ]) wf = pipeline_workflow.workflow_from_pipeline(pipeline, create_directories=False) self.assertEqual(len(wf.jobs), 19) #print(sorted([(d[0].name, d[1].name) for d in wf.dependencies])) self.assertEqual(len(wf.dependencies), 28)
def init_study_config(init_config={}): study_config = StudyConfig('test_study', modules=['AttributesConfig', 'SomaWorkflowConfig'], init_config=init_config) study_config.input_directory = '/tmp/in' study_config.output_directory = '/tmp/out' study_config.attributes_schema_paths \ = study_config.attributes_schema_paths \ + ['capsul.attributes.test.test_attributed_process'] study_config.attributes_schemas['input'] = 'custom_ex' study_config.attributes_schemas['output'] = 'custom_ex' #print('attributes_schema_paths:', study_config.attributes_schema_paths) study_config.path_completion = 'custom_ex' #print('attributes_schema_paths 2:', study_config.attributes_schema_paths) return study_config
def setUp(self): default_config = SortedDictionary( ("use_soma_workflow", True) ) # use a custom temporary soma-workflow dir to avoid concurrent # access problems tmpdb = tempfile.mkstemp('', prefix='soma_workflow') os.close(tmpdb[0]) os.unlink(tmpdb[1]) self.soma_workflow_temp_dir = tmpdb[1] os.mkdir(self.soma_workflow_temp_dir) swf_conf = StringIO.StringIO('[%s]\nSOMA_WORKFLOW_DIR = %s\n' \ % (socket.gethostname(), tmpdb[1])) swconfig.Configuration.search_config_path \ = staticmethod(lambda : swf_conf) self.study_config = StudyConfig(init_config=default_config) self.atomic_pipeline = MyAtomicPipeline() self.composite_pipeline = MyCompositePipeline()
def init_study_config(init_config={}): study_config = StudyConfig('test_study', modules=['AttributesConfig', 'SomaWorkflowConfig'], init_config=init_config) study_config.input_directory = '/tmp/in' study_config.output_directory = '/tmp/out' study_config.attributes_schema_paths.append( 'capsul.attributes.test.test_attributed_process') study_config.attributes_schemas['input'] = 'custom_ex' study_config.attributes_schemas['output'] = 'custom_ex' study_config.path_completion = 'custom_ex' return study_config
def check_call(study_config, batch_file, **kwargs): ''' Equivalent to Python soma.subprocess.check_call for SPM batch ''' check_spm_configuration(study_config) cmd = spm_command(study_config, batch_file) return soma.subprocess.check_call(cmd, **kwargs) def check_output(study_config, command, **kwargs): ''' Equivalent to Python soma.subprocess.check_output for SPM batch ''' check_spm_configuration(study_config) cmd = spm_command(study_config, batch_file) return soma.subprocess.check_output(cmd, **kwargs) if __name__ == '__main__': from capsul.api import StudyConfig from capsul.soma.subprocess.spm import check_call as call_spm import tempfile sc = StudyConfig(spm_directory='/home/yc176684/spm12-standalone-7219') batch = tempfile.NamedTemporaryFile(suffix='.m') batch.write("fprintf(1, '%s', spm('dir'));") batch.flush() call_spm(sc, batch.name)
def main(): ''' Run the :mod:`capsul.process.runprocess` module as a commandline ''' usage = '''Usage: python -m capsul [options] processname [arg1] [arg2] ... [argx=valuex] [argy=valuey] ... Example: python -m capsul threshold ~/data/irm.ima /tmp/th.nii threshold1=80 Named arguments (in the shape argx=valuex) may address sub-processes of a pipeline, using the dot separator: PrepareSubject.t1mri=/home/myself/mymri.nii For a more precise description, please look at the web documentation: http://brainvisa.info/capsul/user_doc/user_guide_tree/index.html ''' # Set up logging on stderr. This must be called before any logging takes # place, to avoid "No handlers could be found for logger" errors. logging.basicConfig() parser = OptionParser(description='Run a single CAPSUL process', usage=usage) group1 = OptionGroup( parser, 'Config', description='Processing configuration, database options') group1.add_option( '--studyconfig', dest='studyconfig', help='load StudyConfig configuration from the given file (JSON)') group1.add_option('-i', '--input', dest='input_directory', help='input data directory (if not specified in ' 'studyconfig file). If not specified neither on the ' 'commandline nor study configfile, taken as the same as ' 'output.') group1.add_option('-o', '--output', dest='output_directory', help='output data directory (if not specified in ' 'studyconfig file). If not specified neither on the ' 'commandline nor study configfile, taken as the same as ' 'input.') parser.add_option_group(group1) group2 = OptionGroup( parser, 'Processing', description='Processing options, distributed execution') group2.add_option('--swf', '--soma_workflow', dest='soma_workflow', default=False, action='store_true', help='use soma_workflow. Soma-Workflow ' 'configuration has to be setup and valid for non-local ' 'execution, and additional file transfer options ' 'may be used. The default is *not* to use SWF and ' 'process mono-processor, sequential execution.') group2.add_option('-r', '--resource_id', dest='resource_id', default=None, help='soma-workflow resource ID, defaults to localhost') group2.add_option('-p', '--password', dest='password', default=None, help='password to access the remote computing resource. ' 'Do not specify it if using a ssh key') group2.add_option('--rsa-pass', dest='rsa_key_pass', default=None, help='RSA key password, for ssh key access') group2.add_option('--queue', dest='queue', default=None, help='Queue to use on the computing resource. If not ' 'specified, use the default queue.') #group2.add_option('--input-processing', dest='input_file_processing', #default=None, help='Input files processing: local_path, ' #'transfer, translate, or translate_shared. The default is ' #'local_path if the computing resource is the localhost, or ' #'translate_shared otherwise.') #group2.add_option('--output-processing', dest='output_file_processing', #default=None, help='Output files processing: local_path, ' #'transfer, or translate. The default is local_path.') group2.add_option('--keep-succeeded-workflow', dest='keep_succeded_workflow', action='store_true', default=False, help='keep the workflow in the computing resource ' 'database after execution. By default it is removed.') group2.add_option('--delete-failed-workflow', dest='delete_failed_workflow', action='store_true', default=False, help='delete the workflow in the computing resource ' 'database after execution, if it has failed. By default ' 'it is kept.') parser.add_option_group(group2) group3 = OptionGroup(parser, 'Iteration', description='Iteration') group3.add_option('-I', '--iterate', dest='iterate_on', action='append', help='Iterate the given process, iterating over the ' 'given parameter(s). Multiple parameters may be ' 'iterated jointly using several -I options. In the ' 'process parameters, values are replaced by lists, all ' 'iterated lists should have the same size.\n' 'Ex:\n' 'python -m capsul -I par_a -I par_c a_process ' 'par_a="[1, 2]" par_b="something" ' 'par_c="[\\"one\\", \\"two\\"]"') parser.add_option_group(group3) group4 = OptionGroup(parser, 'Attributes completion') group4.add_option('-a', '--attribute', dest='attributes', action='append', default=[], help='set completion (including FOM) attribute. ' 'Syntax: attribute=value, value the same syntax as ' 'process parameters (python syntax for lists, for ' 'instance), with proper quotes if needed for shell ' 'escaping.\n' 'Ex: -a acquisition="default" ' '-a subject=\'["s1", "s2"]\'') parser.add_option_group(group4) group5 = OptionGroup(parser, 'Help', description='Help and documentation options') group5.add_option('--process-help', dest='process_help', action='store_true', default=False, help='display specified process help') parser.add_option_group(group5) parser.disable_interspersed_args() (options, args) = parser.parse_args() if options.studyconfig: study_config = StudyConfig(modules=StudyConfig.default_modules + ['FomConfig', 'BrainVISAConfig']) if yaml: scdict = yaml.load(open(options.studyconfig)) else: scdict = json.load(open(options.studyconfig)) study_config.set_study_configuration(scdict) else: study_config = StudyConfig() study_config.read_configuration() if options.input_directory: study_config.input_directory = options.input_directory if options.output_directory: study_config.output_directory = options.output_directory if study_config.output_directory in (None, Undefined) \ and study_config.input_directory not in (None, Undefined): study_config.output_directory = study_config.input_directory if study_config.input_directory in (None, Undefined) \ and study_config.output_directory not in (None, Undefined): study_config.input_directory = study_config.output_directory study_config.somaworkflow_keep_succeeded_workflows \ = options.keep_succeded_workflow study_config.somaworkflow_keep_failed_workflows \ = not options.delete_failed_workflow kwre = re.compile('([a-zA-Z_](\.?[a-zA-Z0-9_])*)\s*=\s*(.*)$') attributes = {} for att in options.attributes: m = kwre.match(att) if m is None: raise SyntaxError('syntax error in attribute definition: %s' % att) attributes[m.group(1)] = convert_commandline_parameter(m.group(3)) args = tuple((convert_commandline_parameter(i) for i in args)) kwargs = {} todel = [] for arg in args: if isinstance(arg, six.string_types): m = kwre.match(arg) if m is not None: kwargs[m.group(1)] = convert_commandline_parameter(m.group(3)) todel.append(arg) args = [arg for arg in args if arg not in todel] if not args: parser.print_usage() sys.exit(2) # get the main process process_name = args[0] args = args[1:] iterated = options.iterate_on try: process = get_process_with_params(process_name, study_config, iterated, attributes, *args, **kwargs) except ProcessParamError as e: print("error: {0}".format(e), file=sys.stderr) sys.exit(1) if options.process_help: process.help() print() completion_engine \ = ProcessCompletionEngine.get_completion_engine(process) attribs = completion_engine.get_attribute_values() aval = attribs.export_to_dict() print('Completion attributes:') print('----------------------') print() print('(note: may differ depending on study config file contents, ' 'completion rules (FOM)...)') print() skipped = set(['generated_by_parameter', 'generated_by_process']) for name, value in six.iteritems(aval): if name in skipped: continue ttype = attribs.trait(name).trait_type.__class__.__name__ if isinstance(attribs.trait(name).trait_type, List): ttype += '(%s)' \ % attribs.trait(name).inner_traits[ 0].trait_type.__class__.__name__ print('%s:' % name, ttype) if value not in (None, Undefined): print(' ', value) print() del aval, attribs, completion_engine, process sys.exit(0) resource_id = options.resource_id password = options.password rsa_key_pass = options.rsa_key_pass queue = options.queue file_processing = [] study_config.use_soma_workflow = options.soma_workflow if options.soma_workflow: file_processing = [None, None] else: file_processing = [None, None] res = run_process_with_distribution( study_config, process, options.soma_workflow, resource_id=resource_id, password=password, rsa_key_pass=rsa_key_pass, queue=queue, input_file_processing=file_processing[0], output_file_processing=file_processing[1]) sys.exit(0)
class TestSomaWorkflow(unittest.TestCase): def setUp(self): default_config = SortedDictionary( ("use_soma_workflow", True) ) self.study_config = StudyConfig(init_config=default_config) self.atomic_pipeline = MyAtomicPipeline() self.composite_pipeline = MyCompositePipeline() def tearDown(self): swm = self.study_config.modules['SomaWorkflowConfig'] swc = swm.get_workflow_controller() if swc is not None: # stop workflow controller and wait for thread termination swc.stop_engine() def test_atomic_dependencies(self): workflow = workflow_from_pipeline(self.atomic_pipeline) dependencies = [(x.name, y.name) for x, y in workflow.dependencies] self.assertTrue(len(dependencies) == 4) self.assertTrue(("node1", "node2") in dependencies) self.assertTrue(("node1", "node3") in dependencies) self.assertTrue(("node2", "node4") in dependencies) self.assertTrue(("node3", "node4") in dependencies) self.assertEqual(workflow.groups, []) def test_atomic_execution(self): self.atomic_pipeline.workflow_ordered_nodes() self.assertIn(self.atomic_pipeline.workflow_repr, ('node1->node3->node2->node4', 'node1->node2->node3->node4')) tmp1 = tempfile.mkstemp('', prefix='capsul_swf') os.write(tmp1[0], 'bidibidi'.encode()) os.close(tmp1[0]) tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out') os.close(tmp2[0]) os.unlink(tmp2[1]) try: self.atomic_pipeline.input_image = tmp1[1] self.atomic_pipeline.output_image = tmp2[1] self.study_config.run(self.atomic_pipeline) self.assertTrue(os.path.exists(tmp2[1])) with open(tmp2[1]) as f: content = f.read() self.assertEqual(content, 'bidibidibidibidi') finally: if os.path.exists(tmp1[1]): os.unlink(tmp1[1]) if os.path.exists(tmp2[1]): os.unlink(tmp2[1]) def test_atomic_execution_kwparams(self): tmp1 = tempfile.mkstemp('', prefix='capsul_swf') os.write(tmp1[0], 'bidibidi'.encode()) os.close(tmp1[0]) tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out') os.close(tmp2[0]) os.unlink(tmp2[1]) try: atomic_pipeline = MyAtomicPipeline() self.study_config.run(atomic_pipeline, input_image=tmp1[1], output_image=tmp2[1]) self.assertTrue(os.path.exists(tmp2[1])) with open(tmp2[1]) as f: content = f.read() self.assertEqual(content, 'bidibidibidibidi') finally: if os.path.exists(tmp1[1]): os.unlink(tmp1[1]) if os.path.exists(tmp2[1]): os.unlink(tmp2[1]) def test_composite_dependencies(self): workflow = workflow_from_pipeline(self.composite_pipeline) dependencies = [(x.name, y.name) for x, y in workflow.dependencies] self.assertEqual(len(dependencies), 8) self.assertEqual(dependencies.count(("node1", "node2")), 1) self.assertEqual(dependencies.count(("node1", "node3")), 2) self.assertEqual(dependencies.count(("node2", "node4")), 1) self.assertEqual(dependencies.count(("node3", "node4")), 2) self.assertEqual(dependencies.count(("node1", "node1")), 1) self.assertEqual(dependencies.count(("node4", "node4")), 1) #self.assertEqual(dependencies.count(("node1", "node2_input")), 1) #self.assertEqual(dependencies.count(("node2_output", "node4")), 1) self.assertTrue(len(workflow.groups) == 1) def test_composite_execution(self): self.composite_pipeline.workflow_ordered_nodes() self.assertTrue(self.composite_pipeline.workflow_repr in ("node1->node3->node2->node4", "node1->node2->node3->node4")) tmp1 = tempfile.mkstemp('', prefix='capsul_swf') os.write(tmp1[0], 'bidibidi'.encode()) os.close(tmp1[0]) tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out') os.close(tmp2[0]) os.unlink(tmp2[1]) try: self.composite_pipeline.input_image = tmp1[1] self.composite_pipeline.output_image = tmp2[1] self.study_config.run(self.composite_pipeline) self.assertTrue(os.path.exists(tmp2[1])) with open(tmp2[1]) as f: content = f.read() self.assertEqual(content, 'bidibidibidibidibidibidi') finally: if os.path.exists(tmp1[1]): os.unlink(tmp1[1]) if os.path.exists(tmp2[1]): os.unlink(tmp2[1])
class TestPipeline(unittest.TestCase): """ Class to test a pipeline with an iterative node """ def setUp(self): """ In the setup construct the pipeline and set some input parameters. """ self.directory = tempfile.mkdtemp(prefix="capsul_test") self.study_config = StudyConfig() # Construct the pipeline self.pipeline = self.study_config.get_process_instance(MyPipeline) # Set some input parameters self.pipeline.input_image = [ os.path.join(self.directory, "toto"), os.path.join(self.directory, "tutu") ] self.pipeline.dynamic_parameter = [3, 1] self.pipeline.other_input = 5 # build a pipeline with dependencies self.small_pipeline \ = self.study_config.get_process_instance(MySmallPipeline) self.small_pipeline.files_to_create = [ os.path.join(self.directory, "toto"), os.path.join(self.directory, "tutu") ] self.small_pipeline.dynamic_parameter = [3, 1] self.small_pipeline.other_input = 5 # build a bigger pipeline with several levels self.big_pipeline \ = self.study_config.get_process_instance(MyBigPipeline) def tearDown(self): swm = self.study_config.modules['SomaWorkflowConfig'] swc = swm.get_workflow_controller() if swc is not None: # stop workflow controller and wait for thread termination swc.stop_engine() if debug: print('directory %s not removed.' % self.directory) else: shutil.rmtree(self.directory) def test_iterative_pipeline_connection(self): """ Test if an iterative process works correctly """ # create inputs for f in self.pipeline.input_image: with open(f, "w") as fobj: fobj.write("input: %s\n" % f) # Test the output connection self.pipeline() if sys.version_info >= (2, 7): self.assertIn( "toto-5.0-3.0", [os.path.basename(f) for f in self.pipeline.output_image]) self.assertIn( "tutu-5.0-1.0", [os.path.basename(f) for f in self.pipeline.output_image]) else: self.assertTrue( "toto-5.0-3.0" in [os.path.basename(f) for f in self.pipeline.output_image]) self.assertTrue( "tutu-5.0-1.0" in [os.path.basename(f) for f in self.pipeline.output_image]) self.assertEqual( self.pipeline.other_output, [self.pipeline.other_input, self.pipeline.other_input]) def test_iterative_pipeline_workflow(self): self.small_pipeline.output_image = [ os.path.join(self.directory, 'toto_out'), os.path.join(self.directory, 'tutu_out') ] self.small_pipeline.other_output = [1., 2.] workflow = pipeline_workflow.workflow_from_pipeline( self.small_pipeline) #expect 2 + 2 (iter) + 2 (barriers) jobs self.assertEqual(len(workflow.jobs), 6) # expect 6 dependencies: # init -> iterative input barrier # iterative output barrier -> end # iterative input barrier -> iterative jobs (2) # iterative jobs -> iterative output barrier (2) self.assertEqual(len(workflow.dependencies), 6) def test_iterative_big_pipeline_workflow(self): self.big_pipeline.files_to_create = [["toto", "tutu"], ["tata", "titi", "tete"]] self.big_pipeline.dynamic_parameter = [[1, 2], [3, 4, 5]] self.big_pipeline.other_input = 5 self.big_pipeline.output_image = [ [ os.path.join(self.directory, 'toto_out'), os.path.join(self.directory, 'tutu_out') ], [ os.path.join(self.directory, 'tata_out'), os.path.join(self.directory, 'titi_out'), os.path.join(self.directory, 'tete_out') ] ] self.big_pipeline.other_output = [[1.1, 2.1], [3.1, 4.1, 5.1]] workflow = pipeline_workflow.workflow_from_pipeline(self.big_pipeline) # expect 6 + 7 + 2 jobs self.assertEqual(len(workflow.jobs), 15) subjects = set() for job in workflow.jobs: if not job.name.startswith('DummyProcess') or '_map' in job.name \ or '_reduce' in job.name: continue param_dict = job.param_dict self.assertEqual(param_dict["other_input"], 5) subject = param_dict['input_image'] subjects.add(subject) if sys.version_info >= (2, 7): self.assertIn(subject, ["toto", "tutu", "tata", "titi", "tete"]) else: self.assertTrue( subject in ["toto", "tutu", "tata", "titi", "tete"]) self.assertEqual(subjects, set(["toto", "tutu", "tata", "titi", "tete"])) def test_iterative_pipeline_workflow_run(self): import soma_workflow.constants as swconstants import soma_workflow.client as swclient self.small_pipeline.output_image = [ os.path.join(self.directory, 'toto_out'), os.path.join(self.directory, 'tutu_out') ] self.small_pipeline.other_output = [1., 2.] workflow = pipeline_workflow.workflow_from_pipeline( self.small_pipeline) swclient.Helper.serialize( os.path.join(self.directory, 'smallpipeline.workflow'), workflow) self.study_config.use_soma_workflow = True #controller = swclient.WorkflowController(config=config) #try: #wf_id = controller.submit_workflow(workflow) print('* running pipeline...') #swclient.Helper.wait_workflow(wf_id, controller) self.study_config.run(self.small_pipeline) print('* finished.') #workflow_status = controller.workflow_status(wf_id) #elements_status = controller.workflow_elements_status(wf_id) #failed_jobs = [element for element in elements_status[0] \ #if element[1] != swconstants.DONE \ #or element[3][0] != swconstants.FINISHED_REGULARLY] #if not debug: #controller.delete_workflow(wf_id) #self.assertTrue(workflow_status == swconstants.WORKFLOW_DONE, #'Workflow did not finish regularly: %s' % workflow_status) #self.assertTrue(len(failed_jobs) == 0, 'Jobs failed: %s' #% failed_jobs) # check output files contents for ifname, fname in zip(self.small_pipeline.files_to_create, self.small_pipeline.output_image): with open(fname) as f: content = f.read() self.assertEqual(content, "file: %s\n" % ifname)
def setUp(self): """ Initialize the TestQCNodes class """ self.pipeline = MyPipeline() self.output_directory = tempfile.mkdtemp() self.study_config = StudyConfig(output_directory=self.output_directory)
def test_leave_one_out_pipeline(self): sc = StudyConfig() pipeline = sc.get_process_instance(PipelineLOO) self._test_loo_pipeline(pipeline)
class TestSomaWorkflow(unittest.TestCase): def setUp(self): default_config = SortedDictionary( ("use_soma_workflow", True) ) # use a custom temporary soma-workflow dir to avoid concurrent # access problems tmpdb = tempfile.mkstemp('', prefix='soma_workflow') os.close(tmpdb[0]) os.unlink(tmpdb[1]) self.soma_workflow_temp_dir = tmpdb[1] os.mkdir(self.soma_workflow_temp_dir) swf_conf = StringIO.StringIO('[%s]\nSOMA_WORKFLOW_DIR = %s\n' \ % (socket.gethostname(), tmpdb[1])) swconfig.Configuration.search_config_path \ = staticmethod(lambda : swf_conf) self.study_config = StudyConfig(init_config=default_config) self.atomic_pipeline = MyAtomicPipeline() self.composite_pipeline = MyCompositePipeline() def tearDown(self): shutil.rmtree(self.soma_workflow_temp_dir) def test_atomic_dependencies(self): workflow = workflow_from_pipeline(self.atomic_pipeline) dependencies = [(x.name, y.name) for x, y in workflow.dependencies] self.assertTrue(len(dependencies) == 4) self.assertTrue(("node1", "node2") in dependencies) self.assertTrue(("node1", "node3") in dependencies) self.assertTrue(("node2", "node4") in dependencies) self.assertTrue(("node3", "node4") in dependencies) self.assertEqual(workflow.groups, []) def test_atomic_execution(self): self.atomic_pipeline.workflow_ordered_nodes() if sys.version_info >= (2, 7): self.assertIn(self.atomic_pipeline.workflow_repr, ('node1->node3->node2->node4', 'node1->node2->node3->node4')) else: # python 2.6 unittest does not have assertIn() self.assertTrue(self.atomic_pipeline.workflow_repr in \ ('node1->node3->node2->node4', 'node1->node2->node3->node4')) tmp1 = tempfile.mkstemp('', prefix='capsul_swf') os.write(tmp1[0], 'bidibidi'.encode()) os.close(tmp1[0]) tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out') os.close(tmp2[0]) os.unlink(tmp2[1]) self.atomic_pipeline.input_image = tmp1[1] self.atomic_pipeline.output_image = tmp2[1] self.study_config.run(self.atomic_pipeline) self.assertTrue(os.path.exists(tmp2[1])) content = open(tmp2[1]).read() self.assertEqual(content, 'bidibidibidibidi') os.unlink(tmp1[1]) os.unlink(tmp2[1]) def test_composite_dependencies(self): workflow = workflow_from_pipeline(self.composite_pipeline) dependencies = [(x.name, y.name) for x, y in workflow.dependencies] self.assertTrue(len(dependencies) == 16) self.assertEqual(dependencies.count(("node1", "node2")), 1) self.assertEqual(dependencies.count(("node1", "node3")), 2) self.assertEqual(dependencies.count(("node2", "node4")), 1) self.assertEqual(dependencies.count(("node3", "node4")), 2) self.assertEqual(dependencies.count(("node1", "node2_input")), 1) self.assertEqual(dependencies.count(("node2_output", "node4")), 1) self.assertTrue(len(workflow.groups) == 1) def test_composite_execution(self): self.composite_pipeline.workflow_ordered_nodes() self.assertTrue(self.composite_pipeline.workflow_repr in ("node1->node3->node2->node4", "node1->node2->node3->node4")) tmp1 = tempfile.mkstemp('', prefix='capsul_swf') os.write(tmp1[0], 'bidibidi'.encode()) os.close(tmp1[0]) tmp2 = tempfile.mkstemp('', prefix='capsul_swf_out') os.close(tmp2[0]) os.unlink(tmp2[1]) self.composite_pipeline.input_image = tmp1[1] self.composite_pipeline.output_image = tmp2[1] self.study_config.run(self.composite_pipeline) self.assertTrue(os.path.exists(tmp2[1])) content = open(tmp2[1]).read() self.assertEqual(content, 'bidibidibidibidibidibidi') os.unlink(tmp1[1]) os.unlink(tmp2[1])
def test_custom_nodes(self): sc = StudyConfig() pipeline = sc.get_process_instance(Pipeline1) self._test_custom_nodes(pipeline)
pipelines = find_pipeline_and_process( os.path.basename(options.module))["pipeline_descs"] logger.info("Found '{0}' pipeline(s) in '{1}'.".format( len(pipelines), options.module)) # Sort pipelines processes # From the pipelines full path 'm1.m2.pipeline' get there module names 'm2' module_names = set([x.split(".")[1] for x in pipelines]) # Sort each pipeline according to its module name. # The result is a dict of the form 'd[m2] = [pipeline1, pipeline2, ...]'. sorted_pipelines = dict((x, []) for x in module_names) for pipeline in pipelines: module_name = pipeline.split(".")[1] sorted_pipelines[module_name].append(pipeline) study_config = StudyConfig(modules=StudyConfig.default_modules + ['FomConfig']) # Generate a png representation of each pipeline. for module_name, module_pipelines in sorted_pipelines.items(): # this docwriter is juste used to manage short names docwriter = PipelineHelpWriter([], short_names=short_names) # Where the documentation will be written: a relative path from the # makefile short_name = docwriter.get_short_name(module_name) outdir = os.path.join(base_outdir, short_name, "schema") if not os.path.isdir(outdir): os.makedirs(outdir) # Go through all pipeline