def execute_task(pckld_task, node_config, updatehash): from socket import gethostname from traceback import format_exc from nipype import config, logging traceback = None result = None import os cwd = os.getcwd() try: config.update_config(node_config) logging.update_logging(config) from pickle import loads task = loads(pckld_task) result = task.run(updatehash=updatehash) except: traceback = format_exc() from pickle import loads task = loads(pckld_task) result = task.result os.chdir(cwd) return result, traceback, gethostname()
def phantoms_wf(options, cfg): import glob import nipype.pipeline.engine as pe from nipype import config, logging from nipype.interfaces import utility as niu from pyacwereg.workflows import evaluation as ev config.update_config(cfg) logging.update_logging(config) grid_size = options.grid_size if len(grid_size) == 1: grid_size = grid_size * 3 bs = ev.bspline(name=options.name, shapes=options.shape, snr_list=options.snr, N=options.repetitions) bs.inputs.inputnode.grid_size = grid_size bs.inputs.inputnode.lo_matrix = options.lo_matrix bs.inputs.inputnode.hi_matrix = options.hi_matrix bs.inputs.inputnode.cortex = options.no_cortex if options.out_csv is None: bs.inputs.inputnode.out_csv = op.join(options.work_dir, bs.name, "results.csv") else: bs.inputs.inputnode.out_csv = options.out_csv return bs
def init(cls): """Set NiPype configurations.""" from nipype import config as ncfg # Configure resource_monitor if cls.resource_monitor: ncfg.update_config({ "monitoring": { "enabled": cls.resource_monitor, "sample_frequency": "0.5", "summary_append": True, } }) ncfg.enable_resource_monitor() # Nipype config (logs and execution) ncfg.update_config({ "execution": { "crashdump_dir": str(execution.log_dir), "crashfile_format": cls.crashfile_format, "get_linked_libs": cls.get_linked_libs, "stop_on_first_crash": cls.stop_on_first_crash, "check_version": False, # disable future telemetry } }) if cls.omp_nthreads is None: cls.omp_nthreads = min( cls.nprocs - 1 if cls.nprocs > 1 else os.cpu_count(), 8)
def init(cls): """Set NiPype configurations.""" from nipype import config as ncfg # Configure resource_monitor if cls.resource_monitor: ncfg.update_config({ 'monitoring': { 'enabled': cls.resource_monitor, 'sample_frequency': '0.5', 'summary_append': True, } }) ncfg.enable_resource_monitor() # Nipype config (logs and execution) ncfg.update_config({ 'execution': { 'crashdump_dir': str(execution.log_dir), 'crashfile_format': cls.crashfile_format, 'get_linked_libs': cls.get_linked_libs, 'stop_on_first_crash': cls.stop_on_first_crash, } }) if cls.omp_nthreads is None: cls.omp_nthreads = min( cls.nprocs - 1 if cls.nprocs > 1 else os.cpu_count(), 8)
def init(cls): """Set NiPype configurations.""" from nipype import config as ncfg # Configure resource_monitor if cls.resource_monitor: ncfg.update_config( { "monitoring": { "enabled": cls.resource_monitor, "sample_frequency": "0.5", "summary_append": True, } } ) ncfg.enable_resource_monitor() # Nipype config (logs and execution) ncfg.update_config( { "execution": { "crashdump_dir": str(execution.log_dir), "crashfile_format": cls.crashfile_format, "get_linked_libs": cls.get_linked_libs, "stop_on_first_crash": cls.stop_on_first_crash, "parameterize_dirs": cls.parameterize_dirs, } } )
def init(cls): """ Set the log level, initialize all loggers into :py:class:`loggers`. * Add new logger levels (25: IMPORTANT, and 15: VERBOSE). * Add a new sub-logger (``cli``). * Logger configuration. """ from nipype import config as ncfg _handler = logging.StreamHandler(stream=sys.stdout) _handler.setFormatter( logging.Formatter(fmt=cls._fmt, datefmt=cls._datefmt)) cls.cli.addHandler(_handler) cls.default.setLevel(execution.log_level) cls.cli.setLevel(execution.log_level) cls.interface.setLevel(execution.log_level) cls.workflow.setLevel(execution.log_level) cls.utils.setLevel(execution.log_level) ncfg.update_config({ "logging": { "log_directory": str(execution.log_dir), "log_to_file": True } })
def test_init_nibetaseries_participant_wf( bids_dir, deriv_dir, sub_fmriprep, sub_metadata, bold_file, preproc_file, sub_events, confounds_file, brainmask_file, atlas_file, atlas_lut, ): output_dir = op.join(str(bids_dir), 'derivatives', 'atlasCorr') work_dir = op.join(str(bids_dir), 'derivatives', 'work') deriv_dir = op.join(str(bids_dir), 'derivatives', 'fmriprep') ncfg.update_config({ 'logging': {'log_directory': work_dir, 'log_to_file': True}, 'execution': {'crashdump_dir': work_dir, 'crashfile_format': 'txt', 'parameterize_dirs': False}, }) test_np_wf = init_nibetaseries_participant_wf( atlas_img=str(atlas_file), atlas_lut=str(atlas_lut), bids_dir=str(bids_dir), derivatives_pipeline_dir=deriv_dir, exclude_description_label=None, hrf_model='spm', high_pass=0.008, output_dir=output_dir, run_label=None, selected_confounds=['WhiteMatter', 'CSF'], session_label=None, smoothing_kernel=None, space_label=None, subject_list=["01"], task_label=None, description_label=None, work_dir=work_dir) assert test_np_wf.run()
def ants_ct_wf(subjects_id, preprocessed_data_dir, working_dir, ds_dir, template_dir, plugin_name): import os from nipype import config from nipype.pipeline.engine import Node, Workflow, MapNode import nipype.interfaces.utility as util import nipype.interfaces.io as nio from nipype.interfaces.freesurfer.utils import ImageInfo ##################################### # GENERAL SETTINGS ##################################### wf = Workflow(name='ants_ct') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True, 'remove_unnecessary_outputs': True, 'job_finished_timeout': 120}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash') ds = Node(nio.DataSink(base_directory=ds_dir), name='ds') ##################################### # GET DATA ##################################### # GET SUBJECT SPECIFIC STRUCTURAL DATA in_data_templates = { 't1w': '{subject_id}/raw_niftis/sMRI/t1w_reoriented.nii.gz', } in_data = Node(nio.SelectFiles(in_data_templates, base_directory=preprocessed_data_dir), name="in_data") in_data.inputs.subject_id = subjects_id # GET NKI ANTs templates ants_templates_templates = { 'brain_template': 'NKI/T_template.nii.gz', 'brain_probability_mask': 'NKI/T_templateProbabilityMask.nii.gz', 'segmentation_priors': 'NKI/Priors/*.nii.gz', 't1_registration_template': 'NKI/T_template_BrainCerebellum.nii.gz' } ants_templates = Node(nio.SelectFiles(ants_templates_templates, base_directory=template_dir), name="ants_templates")
def process(self): """Executes the fMRI pipeline workflow and returns True if successful.""" # Enable the use of the the W3C PROV data model to capture and represent provenance in Nipype # config.enable_provenance() # Process time self.now = datetime.datetime.now().strftime("%Y%m%d_%H%M") cmp_deriv_subject_directory, nipype_deriv_subject_directory, nipype_fmri_pipeline_subject_dir = \ self.init_subject_derivatives_dirs() # Initialization log_file = os.path.join(nipype_fmri_pipeline_subject_dir, "pypeline.log") if os.path.isfile(log_file): os.unlink(log_file) config.update_config({ "logging": { "workflow_level": "INFO", "interface_level": "INFO", "log_directory": nipype_fmri_pipeline_subject_dir, "log_to_file": True, }, "execution": { "remove_unnecessary_outputs": False, "stop_on_first_crash": True, "stop_on_first_rerun": False, "try_hard_link_datasink": True, "use_relative_paths": True, "crashfile_format": "txt", }, }) logging.update_logging(config) iflogger = logging.getLogger("nipype.interface") iflogger.info("**** Processing ****") flow = self.create_pipeline_flow( cmp_deriv_subject_directory=cmp_deriv_subject_directory, nipype_deriv_subject_directory=nipype_deriv_subject_directory, ) flow.write_graph(graph2use="colored", format="svg", simple_form=False) # Create dictionary of arguments passed to plugin_args plugin_args = { 'maxtasksperchild': 1, 'n_procs': self.number_of_cores, 'raise_insufficient': False, } flow.run(plugin="MultiProc", plugin_args=plugin_args) iflogger.info("**** Processing finished ****") return True
def run_command(self, args): """ """ from tempfile import mkdtemp from clinica.pipelines.dwi_preprocessing_multi_shell.dwi_preprocessing_multi_shell_pipeline import DwiPreprocessingMultiShell import os import errno from clinica.iotools.utils.data_handling import create_subs_sess_list from nipype import config cfg = dict(execution={'parameterize_dirs': False}) config.update_config(cfg) if args.subjects_sessions_tsv is None: try: temp_dir = mkdtemp() except OSError as exception: if exception.errno != errno.EEXIST: raise create_subs_sess_list(args.bids_directory, temp_dir) args.subjects_sessions_tsv = os.path.join(temp_dir, 'subjects_sessions_list.tsv') pipeline = DwiPreprocessingNoddi( bids_directory=self.absolute_path(args.bids_directory), caps_directory=self.absolute_path(args.caps_directory), tsv_file=self.absolute_path(args.subjects_sessions_tsv)) pipeline.parameters = { 'epi_param': dict( [ ('echospacing', args.echo_spacing), ('acc_factor', args.acceleration_factor), ('enc_dir', args.phase_encoding_direction), ('epi_factor', args.epi_factor) ]), 'alt_epi_params': dict( [ ('echospacing', args.echo_spacing), ('acc_factor', args.acceleration_factor), ('enc_dir_alt', args.phase_encoding_direction_alternative), ('epi_factor', args.epi_factor) ]) } if args.working_directory is None: args.working_directory = mkdtemp() pipeline.base_dir = self.absolute_path(args.working_directory) if args.n_procs: pipeline.run(plugin='MultiProc', plugin_args={'n_procs': args.n_procs}) else: pipeline.run()
def _create_singleSession(dataDict, master_config, interpMode, pipeline_name): """ create singleSession workflow on a single session This is the main function to call when processing a data set with T1 & T2 data. ExperimentBaseDirectoryPrefix is the base of the directory to place results, T1Images & T2Images are the lists of images to be used in the auto-workup. atlas_fname_wpath is the path and filename of the atlas to use. """ assert 'tissue_classify' in master_config['components'] or \ 'auxlmk' in master_config['components'] or \ 'denoise' in master_config['components'] or \ 'landmark' in master_config['components'] or \ 'segmentation' in master_config['components'] or \ 'malf_2012_neuro' in master_config['components'] from nipype import config, logging config.update_config(master_config) # Set universal pipeline options logging.update_logging(config) from workflows.baseline import generate_single_session_template_WF project = dataDict['project'] subject = dataDict['subject'] session = dataDict['session'] blackListFileName = dataDict['T1s'][0] + '_noDenoise' isBlackList = os.path.isfile(blackListFileName) pname = "{0}_{1}_{2}".format(master_config['workflow_phase'], subject, session) onlyT1 = not (len(dataDict['T2s']) > 0) sessionWorkflow = generate_single_session_template_WF( project, subject, session, onlyT1, master_config, phase=master_config['workflow_phase'], interpMode=interpMode, pipeline_name=pipeline_name, doDenoise=(not isBlackList)) sessionWorkflow.base_dir = master_config['cachedir'] sessionWorkflow_inputsspec = sessionWorkflow.get_node('inputspec') sessionWorkflow_inputsspec.inputs.T1s = dataDict['T1s'] sessionWorkflow_inputsspec.inputs.T2s = dataDict['T2s'] sessionWorkflow_inputsspec.inputs.PDs = dataDict['PDs'] sessionWorkflow_inputsspec.inputs.FLs = dataDict['FLs'] sessionWorkflow_inputsspec.inputs.OTHERs = dataDict['OTs'] return sessionWorkflow
def init(cls): """Set NiPype configurations.""" from nipype import config as ncfg # Nipype config (logs and execution) ncfg.update_config({ "execution": { "crashdump_dir": str(execution.log_dir), "crashfile_format": cls.crashfile_format, "get_linked_libs": cls.get_linked_libs, "stop_on_first_crash": cls.stop_on_first_crash, } })
def ants_ct_wf(subjects_id, preprocessed_data_dir, working_dir, ds_dir, template_dir, plugin_name): import os from nipype import config from nipype.pipeline.engine import Node, Workflow, MapNode import nipype.interfaces.utility as util import nipype.interfaces.io as nio from nipype.interfaces.freesurfer.utils import ImageInfo ##################################### # GENERAL SETTINGS ##################################### wf = Workflow(name='ants_ct') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={ 'stop_on_first_crash': True, 'remove_unnecessary_outputs': True, 'job_finished_timeout': 120 }) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join( working_dir, 'crash') ds = Node(nio.DataSink(base_directory=ds_dir), name='ds') ##################################### # GET DATA ##################################### # GET SUBJECT SPECIFIC STRUCTURAL DATA in_data_templates = { 't1w': '{subject_id}/raw_niftis/sMRI/t1w_reoriented.nii.gz', } in_data = Node(nio.SelectFiles(in_data_templates, base_directory=preprocessed_data_dir), name="in_data") in_data.inputs.subject_id = subjects_id # GET NKI ANTs templates ants_templates_templates = { 'brain_template': 'NKI/T_template.nii.gz', 'brain_probability_mask': 'NKI/T_templateProbabilityMask.nii.gz', 'segmentation_priors': 'NKI/Priors/*.nii.gz', 't1_registration_template': 'NKI/T_template_BrainCerebellum.nii.gz' } ants_templates = Node(nio.SelectFiles(ants_templates_templates, base_directory=template_dir), name="ants_templates")
def _create_singleSession(dataDict, master_config, interpMode, pipeline_name): """ create singleSession workflow on a single session This is the main function to call when processing a data set with T1 & T2 data. ExperimentBaseDirectoryPrefix is the base of the directory to place results, T1Images & T2Images are the lists of images to be used in the auto-workup. atlas_fname_wpath is the path and filename of the atlas to use. """ assert 'tissue_classify' in master_config['components'] or \ 'auxlmk' in master_config['components'] or \ 'denoise' in master_config['components'] or \ 'landmark' in master_config['components'] or \ 'segmentation' in master_config['components'] or \ 'malf_2012_neuro' in master_config['components'] from nipype import config, logging config.update_config(master_config) # Set universal pipeline options logging.update_logging(config) from workflows.baseline import generate_single_session_template_WF project = dataDict['project'] subject = dataDict['subject'] session = dataDict['session'] blackListFileName = dataDict['T1s'][0] + '_noDenoise' isBlackList = os.path.isfile(blackListFileName) pname = "{0}_{1}_{2}".format(master_config['workflow_phase'], subject, session) onlyT1 = not (len(dataDict['T2s']) > 0) if onlyT1: print "T1 Only processing starts ..." else: print "Multimodal processing starts ..." sessionWorkflow = generate_single_session_template_WF(project, subject, session, onlyT1, master_config, phase=master_config['workflow_phase'], interpMode=interpMode, pipeline_name=pipeline_name, doDenoise=(not isBlackList)) sessionWorkflow.base_dir = master_config['cachedir'] sessionWorkflow_inputsspec = sessionWorkflow.get_node('inputspec') sessionWorkflow_inputsspec.inputs.T1s = dataDict['T1s'] sessionWorkflow_inputsspec.inputs.T2s = dataDict['T2s'] sessionWorkflow_inputsspec.inputs.PDs = dataDict['PDs'] sessionWorkflow_inputsspec.inputs.FLs = dataDict['FLs'] sessionWorkflow_inputsspec.inputs.OTHERs = dataDict['OTs'] return sessionWorkflow
def RunConnectivityWorkflow(path, outDir, workdir, conf_file=None): if conf_file is not None: res = open(conf_file, 'r').read() conf = json.loads(res) config.update_config(conf) plugin = config.get("execution", "plugin") plugin_args = config.get("execution", "plugin_args") if plugin_args is not None: plugin_args = eval(plugin_args) wf = BuildConnectivityWorkflowSurface(path, outDir) wf.base_dir = workdir wf.run(plugin=plugin, plugin_args=plugin_args)
def execute_task(pckld_task, node_config, updatehash): from socket import gethostname from traceback import format_exc from nipype import config, logging traceback=None result=None try: config.update_config(node_config) logging.update_logging(config) from cPickle import loads task = loads(pckld_task) result = task.run(updatehash=updatehash) except: traceback = format_exc() result = task.result return result, traceback, gethostname()
def prep_logging(opts, output_folder): cli_file = f'{output_folder}/rabies_{opts.rabies_stage}.pkl' if os.path.isfile(cli_file): raise ValueError(f""" A previous run was indicated by the presence of {cli_file}. This can lead to inconsistencies between previous outputs and the log files. To prevent this, you are required to manually remove {cli_file}, and we recommend also removing previous datasinks from the {opts.rabies_stage} RABIES step. """) with open(cli_file, 'wb') as handle: pickle.dump(opts, handle, protocol=pickle.HIGHEST_PROTOCOL) # remove old versions of the log if already existing log_path = f'{output_folder}/rabies_{opts.rabies_stage}.log' if os.path.isfile(log_path): os.remove(log_path) config.update_config({'logging': {'log_directory': output_folder, 'log_to_file': True}}) # setting workflow logging level if opts.verbose==0: level="WARNING" elif opts.verbose==1: level="INFO" elif opts.verbose<=2: level="DEBUG" config.enable_debug_mode() else: raise ValueError(f"--verbose must be provided an integer of 0 or above. {opts.verbose} was provided instead.") # nipype has hard-coded 'nipype.log' filename; we rename it after it is created, and change the handlers logging.update_logging(config) os.rename(f'{output_folder}/pypeline.log', log_path) # change the handlers path to the desired file for logger in logging.loggers.keys(): log = logging.getLogger(logger) handler = log.handlers[0] handler.baseFilename = log_path # set the defined level of verbose log = logging.getLogger('nipype.workflow') log.setLevel(level) log.debug('Debug ON') return log
def test_filters_init_nibetaseries_participant_wf( bids_dir, deriv_dir, sub_fmriprep, sub_metadata, bold_file, preproc_file, sub_events, confounds_file, brainmask_file, session_label, task_label, run_label, space_label, description_label): output_dir = op.join(str(bids_dir), 'derivatives', 'atlasCorr') work_dir = op.join(str(bids_dir), 'derivatives', 'work') deriv_dir = op.join(str(bids_dir), 'derivatives', 'fmriprep') ncfg.update_config({ 'logging': {'log_directory': work_dir, 'log_to_file': True}, 'execution': {'crashdump_dir': work_dir, 'crashfile_format': 'txt', 'parameterize_dirs': False}, }) with pytest.raises(ValueError) as val_err: init_nibetaseries_participant_wf( estimator='lsa', fir_delays=None, atlas_img=None, atlas_lut=None, bids_dir=str(bids_dir), database_path=None, derivatives_pipeline_dir=deriv_dir, exclude_description_label=None, hrf_model='spm', high_pass=0.008, norm_betas=False, output_dir=output_dir, return_residuals=False, run_label=run_label, selected_confounds=['white_matter', 'csf'], session_label=session_label, signal_scaling=False, smoothing_kernel=None, space_label=space_label, subject_list=["01"], task_label=task_label, description_label=description_label, work_dir=work_dir) assert "could not find preprocessed outputs:" in str(val_err.value)
def test_valid_init_nibetaseries_participant_wf( bids_dir, deriv_dir, sub_fmriprep, sub_top_metadata, bold_file, preproc_file, sub_events, confounds_file, brainmask_file, atlas_file, atlas_lut, bids_db_file, estimator, fir_delays, hrf_model, signal_scaling, norm_betas): output_dir = op.join(str(bids_dir), 'derivatives', 'atlasCorr') work_dir = op.join(str(bids_dir), 'derivatives', 'work') deriv_dir = op.join(str(bids_dir), 'derivatives', 'fmriprep') ncfg.update_config({ 'logging': {'log_directory': work_dir, 'log_to_file': True}, 'execution': {'crashdump_dir': work_dir, 'crashfile_format': 'txt', 'parameterize_dirs': False}, }) test_np_wf = init_nibetaseries_participant_wf( estimator=estimator, fir_delays=fir_delays, atlas_img=str(atlas_file), atlas_lut=str(atlas_lut), bids_dir=str(bids_dir), database_path=str(bids_db_file), derivatives_pipeline_dir=deriv_dir, exclude_description_label=None, hrf_model=hrf_model, high_pass=0.008, norm_betas=norm_betas, output_dir=output_dir, return_residuals=False, run_label=None, selected_confounds=['white_matter', 'csf'], session_label=None, signal_scaling=signal_scaling, smoothing_kernel=None, space_label=None, subject_list=["01"], task_label=None, description_label=None, work_dir=work_dir) assert test_np_wf.run()
def main(): """Entry point""" from nipype import config, logging from regseg.workflows.phantoms import phantoms_wf options = get_parser().parse_args() # Setup multiprocessing nthreads = options.nthreads if nthreads == 0: from multiprocessing import cpu_count nthreads = cpu_count() cfg = {} cfg['plugin'] = 'Linear' if nthreads > 1: cfg['plugin'] = 'MultiProc' cfg['plugin_args'] = {'n_proc': nthreads} # Setup work_dir if not op.exists(options.work_dir): os.makedirs(options.work_dir) # Setup logging dir log_dir = op.abspath('logs') cfg['logging'] = {'log_directory': log_dir, 'log_to_file': True} if not op.exists(log_dir): os.makedirs(log_dir) config.update_config(cfg) logging.update_logging(config) wf = phantoms_wf(options, cfg) wf.base_dir = options.work_dir wf.write_graph(graph2use='hierarchical', format='pdf', simple_form=True) wf.run()
] sessions = ['COND'] ############################################################################### wf = pe.Workflow(name='QC') wf.base_dir = wd wf.config['execution']['crashdump_dir'] = wf.base_dir + "/crash_files" nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={ 'stop_on_first_crash': False, 'remove_unnecessary_outputs': False, 'job_finished_timeout': 120 }) config.update_config(nipype_cfg) #generate distributions per session subjects2 = [ '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34' ] def get_lists_of_files(data_dir, session, subjects): tsnr_files = [ data_dir + "/MPP/%s/%s/TEMP/moco_tSNR.nii.gz" % (subject, session) for subject in subjects ]
log_dir = os.path.join(conf.output_dir, 'logs', 'group_analysis', resource, 'model_%s' % (conf.model_name)) if not os.path.exists(log_dir): os.makedirs(log_dir) else: print "log_dir already exist" # enable logging from nipype import config from nipype import logging config.update_config( {'logging': { 'log_directory': log_dir, 'log_to_file': True }}) # Temporarily disable until solved #logging.update_logging(config) iflogger = logging.getLogger('interface') ''' create the list of paths to all output files to go to model ''' # create the 'ordered_paths' list, which is a list of all of the # output paths of the output files being included in the current # group-level analysis model # 'ordered_paths' is later connected to the 'zmap_files' input # of the group analysis workflow - the files listed in this list # are merged into the merged 4D file that goes into group analysis
def process(self): # Process time now = datetime.datetime.now().strftime("%Y%m%d_%H%M") # Initialization if os.path.exists(os.path.join(self.base_directory,"LOG","pypeline.log")): os.unlink(os.path.join(self.base_directory,"LOG","pypeline.log")) config.update_config({'logging': {'log_directory': os.path.join(self.base_directory,"LOG"), 'log_to_file': True}, 'execution': {} }) logging.update_logging(config) flow = pe.Workflow(name='diffusion_pipeline', base_dir=os.path.join(self.base_directory,'NIPYPE')) iflogger = logging.getLogger('interface') # Data import datasource = pe.Node(interface=nio.DataGrabber(outfields = ['diffusion','T1','T2']), name='datasource') datasource.inputs.base_directory = os.path.join(self.base_directory,'NIFTI') datasource.inputs.template = '*' datasource.inputs.raise_on_empty = False datasource.inputs.field_template = dict(diffusion=self.global_conf.imaging_model+'.nii.gz',T1='T1.nii.gz',T2='T2.nii.gz') # Data sinker for output sinker = pe.Node(nio.DataSink(), name="sinker") sinker.inputs.base_directory = os.path.join(self.base_directory, "RESULTS") # Clear previous outputs self.clear_stages_outputs() if self.stages['Preprocessing'].enabled: preproc_flow = self.create_stage_flow("Preprocessing") flow.connect([ (datasource,preproc_flow,[("diffusion","inputnode.diffusion")]), ]) if self.stages['Segmentation'].enabled: if self.stages['Segmentation'].config.seg_tool == "Freesurfer": if self.stages['Segmentation'].config.use_existing_freesurfer_data == False: self.stages['Segmentation'].config.freesurfer_subjects_dir = os.path.join(self.base_directory) self.stages['Segmentation'].config.freesurfer_subject_id = os.path.join(self.base_directory,'FREESURFER') if (not os.path.exists(os.path.join(self.base_directory,'NIPYPE/diffusion_pipeline/segmentation_stage/reconall/result_reconall.pklz'))) and os.path.exists(os.path.join(self.base_directory,'FREESURFER')): shutil.rmtree(os.path.join(self.base_directory,'FREESURFER')) seg_flow = self.create_stage_flow("Segmentation") if self.stages['Segmentation'].config.seg_tool == "Freesurfer": flow.connect([(datasource,seg_flow, [('T1','inputnode.T1')])]) if self.stages['Parcellation'].enabled: parc_flow = self.create_stage_flow("Parcellation") if self.stages['Segmentation'].config.seg_tool == "Freesurfer": flow.connect([(seg_flow,parc_flow, [('outputnode.subjects_dir','inputnode.subjects_dir'), ('outputnode.subject_id','inputnode.subject_id')]), ]) else: flow.connect([ (seg_flow,parc_flow,[("outputnode.custom_wm_mask","inputnode.custom_wm_mask")]) ]) if self.stages['Registration'].enabled: reg_flow = self.create_stage_flow("Registration") flow.connect([ (datasource,reg_flow,[('T1','inputnode.T1'),('T2','inputnode.T2')]), (preproc_flow,reg_flow, [('outputnode.diffusion_preproc','inputnode.diffusion')]), (parc_flow,reg_flow, [('outputnode.wm_mask_file','inputnode.wm_mask'),('outputnode.roi_volumes','inputnode.roi_volumes')]), ]) if self.stages['Registration'].config.registration_mode == "BBregister (FS)": flow.connect([ (seg_flow,reg_flow, [('outputnode.subjects_dir','inputnode.subjects_dir'), ('outputnode.subject_id','inputnode.subject_id')]), ]) if self.stages['Diffusion'].enabled: diff_flow = self.create_stage_flow("Diffusion") flow.connect([ (preproc_flow,diff_flow, [('outputnode.diffusion_preproc','inputnode.diffusion')]), (reg_flow,diff_flow, [('outputnode.wm_mask_registered','inputnode.wm_mask_registered')]), (reg_flow,diff_flow,[('outputnode.roi_volumes_registered','inputnode.roi_volumes')]) ]) if self.stages['Connectome'].enabled: if self.stages['Diffusion'].config.processing_tool == 'FSL': self.stages['Connectome'].config.probtrackx = True con_flow = self.create_stage_flow("Connectome") flow.connect([ (parc_flow,con_flow, [('outputnode.parcellation_scheme','inputnode.parcellation_scheme')]), (diff_flow,con_flow, [('outputnode.track_file','inputnode.track_file'),('outputnode.gFA','inputnode.gFA'), ('outputnode.roi_volumes','inputnode.roi_volumes_registered'), ('outputnode.skewness','inputnode.skewness'),('outputnode.kurtosis','inputnode.kurtosis'), ('outputnode.P0','inputnode.P0')]), (con_flow,sinker, [('outputnode.connectivity_matrices',now+'.connectivity_matrices')]) ]) if self.stages['Parcellation'].config.parcellation_scheme == "Custom": flow.connect([(parc_flow,con_flow, [('outputnode.atlas_info','inputnode.atlas_info')])]) iflogger.info("**** Processing ****") if(self.number_of_cores != 1): flow.run(plugin='MultiProc', plugin_args={'n_procs' : self.number_of_cores}) else: flow.run() self.fill_stages_outputs() # Clean undesired folders/files rm_file_list = ['rh.EC_average','lh.EC_average','fsaverage'] for file_to_rm in rm_file_list: if os.path.exists(os.path.join(self.base_directory,file_to_rm)): os.remove(os.path.join(self.base_directory,file_to_rm)) # copy .ini and log file outdir = os.path.join(self.base_directory,"RESULTS",now) if not os.path.exists(outdir): os.makedirs(outdir) shutil.copy(self.config_file,outdir) shutil.copy(os.path.join(self.base_directory,'LOG','pypeline.log'),outdir) iflogger.info("**** Processing finished ****") return True,'Processing sucessful'
def build_functional_temporal_workflow(resource_pool, config, subject_info, \ run_name, site_name=None): # build pipeline for each subject, individually # ~ 5 min 45 sec per subject # (roughly 345 seconds) import os import sys import nipype.interfaces.io as nio import nipype.pipeline.engine as pe import nipype.interfaces.utility as util import nipype.interfaces.fsl.maths as fsl import glob import yaml import time from time import strftime from nipype import config as nyconfig from nipype import logging logger = logging.getLogger('workflow') sub_id = str(subject_info[0]) if subject_info[1]: session_id = str(subject_info[1]) else: session_id = "session_0" if subject_info[2]: scan_id = str(subject_info[2]) else: scan_id = "scan_0" # define and create the output directory output_dir = os.path.join(config["output_directory"], run_name, \ sub_id, session_id, scan_id) try: os.makedirs(output_dir) except: if not os.path.isdir(output_dir): err = "[!] Output directory unable to be created.\n" \ "Path: %s\n\n" % output_dir raise Exception(err) else: pass log_dir = output_dir # set up logging nyconfig.update_config({'logging': {'log_directory': log_dir, 'log_to_file': True}}) logging.update_logging(nyconfig) # take date+time stamp for run identification purposes unique_pipeline_id = strftime("%Y%m%d%H%M%S") pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S") pipeline_start_time = time.time() logger.info(pipeline_start_stamp) logger.info("Contents of resource pool:\n" + str(resource_pool)) logger.info("Configuration settings:\n" + str(config)) # for QAP spreadsheet generation only config["subject_id"] = sub_id config["session_id"] = session_id config["scan_id"] = scan_id config["run_name"] = run_name if site_name: config["site_name"] = site_name workflow = pe.Workflow(name=scan_id) workflow.base_dir = os.path.join(config["working_directory"], sub_id, \ session_id) # set up crash directory workflow.config['execution'] = \ {'crashdump_dir': config["output_directory"]} # update that resource pool with what's already in the output directory for resource in os.listdir(output_dir): if os.path.isdir(os.path.join(output_dir,resource)) and resource not in resource_pool.keys(): resource_pool[resource] = glob.glob(os.path.join(output_dir, \ resource, "*"))[0] # resource pool check invalid_paths = [] for resource in resource_pool.keys(): if not os.path.isfile(resource_pool[resource]): invalid_paths.append((resource, resource_pool[resource])) if len(invalid_paths) > 0: err = "\n\n[!] The paths provided in the subject list to the " \ "following resources are not valid:\n" for path_tuple in invalid_paths: err = err + path_tuple[0] + ": " + path_tuple[1] + "\n" err = err + "\n\n" raise Exception(err) # start connecting the pipeline if "qap_functional_temporal" not in resource_pool.keys(): from qap.qap_workflows import qap_functional_temporal_workflow workflow, resource_pool = \ qap_functional_temporal_workflow(workflow, resource_pool, config) # set up the datasinks new_outputs = 0 if "write_all_outputs" not in config.keys(): config["write_all_outputs"] = False if config["write_all_outputs"] == True: for output in resource_pool.keys(): # we use a check for len()==2 here to select those items in the # resource pool which are tuples of (node, node_output), instead # of the items which are straight paths to files # resource pool items which are in the tuple format are the # outputs that have been created in this workflow because they # were not present in the subject list YML (the starting resource # pool) and had to be generated if len(resource_pool[output]) == 2: ds = pe.Node(nio.DataSink(), name='datasink_%s' % output) ds.inputs.base_directory = output_dir node, out_file = resource_pool[output] workflow.connect(node, out_file, ds, output) new_outputs += 1 else: # write out only the output CSV (default) output = "qap_functional_temporal" if len(resource_pool[output]) == 2: ds = pe.Node(nio.DataSink(), name='datasink_%s' % output) ds.inputs.base_directory = output_dir node, out_file = resource_pool[output] workflow.connect(node, out_file, ds, output) new_outputs += 1 # run the pipeline (if there is anything to do) if new_outputs > 0: workflow.write_graph(dotfilename=os.path.join(output_dir, run_name + \ ".dot"), \ simple_form=False) workflow.run(plugin='MultiProc', plugin_args= \ {'n_procs': config["num_cores_per_subject"]}) else: print "\nEverything is already done for subject %s." % sub_id # Remove working directory when done if config["write_all_outputs"] == False: try: work_dir = os.path.join(workflow.base_dir, scan_id) if os.path.exists(work_dir): import shutil shutil.rmtree(work_dir) except: print "Couldn\'t remove the working directory!" pass pipeline_end_stamp = strftime("%Y-%m-%d_%H:%M:%S") pipeline_end_time = time.time() logger.info("Elapsed time (minutes) since last start: %s" \ % ((pipeline_end_time - pipeline_start_time)/60)) logger.info("Pipeline end time: %s" % pipeline_end_stamp) return workflow
def run_workflow(csv_file, use_pbs, contrasts_name, template): workflow = pe.Workflow(name='run_level1flow') workflow.base_dir = os.path.abspath('./workingdirs') from nipype import config, logging config.update_config({ 'logging': { 'log_directory': os.path.join(workflow.base_dir, 'logs'), 'log_to_file': True, 'workflow_level': 'DEBUG', 'interface_level': 'DEBUG', } }) logging.update_logging(config) config.enable_debug_mode() # redundant with enable_debug_mode() ... workflow.stop_on_first_crash = True workflow.remove_unnecessary_outputs = False workflow.keep_inputs = True workflow.hash_method = 'content' """ Setup the contrast structure that needs to be evaluated. This is a list of lists. The inner list specifies the contrasts and has the following format: [Name,Stat,[list of condition names],[weights on those conditions]. The condition names must match the `names` listed in the `evt_info` function described above. """ try: import importlib mod = importlib.import_module('contrasts.' + contrasts_name) contrasts = mod.contrasts # event_names = mod.event_names except ImportError: raise RuntimeError('Unknown contrasts: %s. Must exist as a Python' ' module in contrasts directory!' % contrasts_name) modelfit = create_workflow(contrasts) import bids_templates as bt inputnode = pe.Node(niu.IdentityInterface(fields=[ 'subject_id', 'session_id', 'run_id', ]), name='input') assert csv_file is not None, "--csv argument must be defined!" reader = niu.CSVReader() reader.inputs.header = True reader.inputs.in_file = csv_file out = reader.run() subject_list = out.outputs.subject session_list = out.outputs.session run_list = out.outputs.run inputnode.iterables = [ ('subject_id', subject_list), ('session_id', session_list), ('run_id', run_list), ] inputnode.synchronize = True templates = { 'funcs': 'derivatives/featpreproc/highpassed_files/sub-{subject_id}/ses-{session_id}/func/' 'sub-{subject_id}_ses-{session_id}_*_run-{run_id}*_bold_res-1x1x1_preproc_*.nii.gz', # 'funcmasks': # 'featpreproc/func_unwarp/sub-{subject_id}/ses-{session_id}/func/' # 'sub-{subject_id}_ses-{session_id}_*_run-{run_id}*_bold_res-1x1x1_preproc' # '_mc_unwarped.nii.gz', 'highpass': '******' 'sub-{subject_id}_ses-{session_id}_*_run-{run_id}_bold_res-1x1x1_preproc_*.nii.gz', 'motion_parameters': 'derivatives/featpreproc/motion_corrected/sub-{subject_id}/ses-{session_id}/func/' 'sub-{subject_id}_ses-{session_id}_*_run-{run_id}_bold_res-1x1x1_preproc.param.1D', 'motion_outlier_files': 'derivatives/featpreproc/motion_outliers/sub-{subject_id}/ses-{session_id}/func/' 'art.sub-{subject_id}_ses-{session_id}_*_run-{run_id}_bold_res-1x1x1_preproc_mc' '_maths_outliers.txt', 'event_log': 'sub-{subject_id}/ses-{session_id}/func/' # 'sub-{subject_id}_ses-{session_id}*_bold_res-1x1x1_preproc' 'sub-{subject_id}_ses-{session_id}*run-{run_id}*' # '.nii.gz', '_events.tsv', 'ref_func': 'derivatives/featpreproc/reference/func/*.nii.gz', 'ref_funcmask': 'derivatives/featpreproc/reference/func_mask/*.nii.gz', } inputfiles = pe.Node(nio.SelectFiles(templates, base_directory=data_dir), name='in_files') workflow.connect([ (inputnode, inputfiles, [ ('subject_id', 'subject_id'), ('session_id', 'session_id'), ('run_id', 'run_id'), ]), ]) join_input = pe.JoinNode( niu.IdentityInterface(fields=[ # 'subject_id', # 'session_id', # 'run_id', 'funcs', 'highpass', 'motion_parameters', 'motion_outlier_files', 'event_log', 'ref_func', 'ref_funcmask', ]), joinsource='input', joinfield=[ 'funcs', 'highpass', 'motion_parameters', 'motion_outlier_files', 'event_log', ], # unique=True, name='join_input') workflow.connect([ (inputfiles, join_input, [ ('funcs', 'funcs'), ('highpass', 'highpass'), ('motion_parameters', 'motion_parameters'), ('motion_outlier_files', 'motion_outlier_files'), ('event_log', 'event_log'), ('ref_func', 'ref_func'), ('ref_funcmask', 'ref_funcmask'), ]), (join_input, modelfit, [ ('funcs', 'inputspec.funcs'), ('highpass', 'inputspec.highpass'), ('motion_parameters', 'inputspec.motion_parameters'), ('motion_outlier_files', 'inputspec.motion_outlier_files'), ('event_log', 'inputspec.event_log'), ('ref_func', 'inputspec.ref_func'), ('ref_funcmask', 'inputspec.ref_funcmask'), ]), ]) modelfit.inputs.inputspec.fwhm = 2.0 modelfit.inputs.inputspec.highpass = 50 modelfit.write_graph(simple_form=True) modelfit.write_graph(graph2use='orig', format='png', simple_form=True) # modelfit.write_graph(graph2use='detailed', format='png', simple_form=False) workflow.stop_on_first_crash = True workflow.keep_inputs = True workflow.remove_unnecessary_outputs = False workflow.write_graph(simple_form=True) workflow.write_graph(graph2use='colored', format='png', simple_form=True) # workflow.write_graph(graph2use='detailed', format='png', simple_form=False) if use_pbs: workflow.run(plugin='PBS', plugin_args={'template': os.path.expanduser(template)}) else: workflow.run()
def test_nuisance(): # from CPAC.nuisance import create_nuisance # cn = create_nuisance() # subjects_list = open('/home/data/Projects/ADHD200/adhd200_sublist_for_basc_withGSR').readlines() # subjects_list = [ subject.strip() for subject in subjects_list ] # # subject = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/funcpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/func_scale/mapflow/_func_scale0/lfo_3dc_RPI_3dv_3dc_maths.nii.gz' # csf_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/segpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_csf_threshold_0.4/seg_mask/mapflow/_seg_mask0/segment_prob_0_flirt_maths_maths_maths.nii.gz' # wm_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/segpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_wm_threshold_0.66/seg_mask1/mapflow/_seg_mask10/segment_prob_2_flirt_maths_maths_maths.nii.gz' # gm_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/segpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_gm_threshold_0.2/seg_mask2/mapflow/_seg_mask20/segment_prob_1_flirt_maths_maths_maths.nii.gz' # motion_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/funcpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/func_volreg_1/mapflow/_func_volreg_10/lfo_3dc_RPI_3dv1D.1D' # # stor = {'compcor' : True, # 'wm' : True, # 'csf' : True, # 'gm' : True, # 'global' : True, # 'pc1' : True, # 'motion' : True, # 'linear' : True, # 'quadratic' : True} # # stor2 = {'compcor' : True, # 'wm' : True, # 'csf' : True, # 'gm' : True, # 'global' : False, # 'pc1' : True, # 'motion' : False, # 'linear' : True, # 'quadratic' : False} # # cn.inputs.inputspec.subject = subject # cn.inputs.inputspec.gm_mask = gm_file # cn.inputs.inputspec.wm_mask = wm_file # cn.inputs.inputspec.csf_mask = csf_file # cn.inputs.inputspec.motion_components = motion_file # cn.get_node('residuals').iterables = ('selector',[stor, stor2]) # cn.inputs.inputspec.compcor_ncomponents = 5 # # cn.run(plugin='MultiProc', plugin_args={'n_procs' : 2}) from nipype import config config.update_config({'execution': {'remove_unnecessary_outputs':False}}) from CPAC.nuisance import create_nuisance cn = create_nuisance() stor = {'compcor' : True, 'wm' : True, 'csf' : True, 'gm' : True, 'global' : True, 'pc1' : True, 'motion' : True, 'linear' : True, 'quadratic' : True} cn.inputs.inputspec.selector = stor cn.inputs.inputspec.compcor_ncomponents = 5 cn.inputs.inputspec.motion_components = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/movement_parameters/_scan_rest_1_rest/rest_3dc_RPI_3dv1D.1D' cn.inputs.inputspec.func_to_anat_linear_xfm = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/functional_to_anat_linear_xfm/_scan_rest_1_rest/rest_3dc_RPI_3dv_3dc_3dT_flirt.mat' cn.inputs.inputspec.mni_to_anat_linear_xfm = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/mni_to_anatomical_linear_xfm/mprage_RPI_3dc_flirt_inv.mat' cn.inputs.inputspec.gm_mask = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/anatomical_gm_mask/_gm_threshold_0.2/segment_prob_1_maths_maths_maths.nii.gz' cn.inputs.inputspec.csf_mask = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/anatomical_csf_mask/_csf_threshold_0.4/segment_prob_0_maths_maths_maths.nii.gz' cn.inputs.inputspec.wm_mask = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/anatomical_wm_mask/_wm_threshold_0.66/segment_prob_2_maths_maths_maths.nii.gz' cn.inputs.inputspec.harvard_oxford_mask = '/usr/share/fsl/4.1/data/atlases/HarvardOxford/HarvardOxford-sub-maxprob-thr25-2mm.nii.gz' cn.inputs.inputspec.subject = '/home/data/PreProc/ABIDE_CPAC_test_1/pipeline_0/0050102_session_1/preprocessed/_scan_rest_1_rest/rest_3dc_RPI_3dv_3dc_maths.nii.gz' cn.base_dir = '/home/bcheung/cn_run'
def learning_prepare_data_wf(working_dir, ds_dir, template_lookup_dict, behav_file, qc_file, in_data_name_list, data_lookup_dict, use_n_procs, plugin_name): import os from nipype import config from nipype.pipeline.engine import Node, Workflow import nipype.interfaces.utility as util import nipype.interfaces.io as nio from prepare_data_utils import vectorize_and_aggregate from itertools import chain # ensure in_data_name_list is list of lists in_data_name_list = [i if type(i) == list else [i] for i in in_data_name_list] in_data_name_list_unique = list(set(chain.from_iterable(in_data_name_list))) ##################################### # GENERAL SETTINGS ##################################### wf = Workflow(name='learning_prepare_data_wf') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': False, 'remove_unnecessary_outputs': False, 'job_finished_timeout': 120, 'hash_method': 'timestamp'}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash') ds = Node(nio.DataSink(), name='ds') ds.inputs.base_directory = os.path.join(ds_dir, 'group_learning_prepare_data') ds.inputs.regexp_substitutions = [ # ('subject_id_', ''), ('_parcellation_', ''), ('_bp_freqs_', 'bp_'), ('_extraction_method_', ''), ('_subject_id_[A0-9]*/', '') ] ds_X = Node(nio.DataSink(), name='ds_X') ds_X.inputs.base_directory = os.path.join(ds_dir, 'vectorized_aggregated_data') ds_pdf = Node(nio.DataSink(), name='ds_pdf') ds_pdf.inputs.base_directory = os.path.join(ds_dir, 'pdfs') ds_pdf.inputs.parameterization = False ##################################### # SET ITERATORS ##################################### # SUBJECTS ITERATOR in_data_name_infosource = Node(util.IdentityInterface(fields=['in_data_name']), name='in_data_name_infosource') in_data_name_infosource.iterables = ('in_data_name', in_data_name_list_unique) mulitmodal_in_data_name_infosource = Node(util.IdentityInterface(fields=['multimodal_in_data_name']), name='mulitmodal_in_data_name_infosource') mulitmodal_in_data_name_infosource.iterables = ('multimodal_in_data_name', in_data_name_list) ############################################################################################################### # GET SUBJECTS INFO # create subjects list based on selection criteria def create_df_fct(behav_file, qc_file): import pandas as pd import os df = pd.read_pickle(behav_file) qc = pd.read_pickle(qc_file) df_all = qc.join(df, how='inner') assert df_all.index.is_unique, 'duplicates in df index. fix before cont.' df_all_subjects_pickle_file = os.path.abspath('df_all.pkl') df_all.to_pickle(df_all_subjects_pickle_file) full_subjects_list = df_all.index.values return df_all_subjects_pickle_file, full_subjects_list create_df = Node(util.Function(input_names=['behav_file', 'qc_file'], output_names=['df_all_subjects_pickle_file', 'full_subjects_list'], function=create_df_fct), name='create_df') create_df.inputs.behav_file = behav_file create_df.inputs.qc_file = qc_file ############################################################################################################### # CREAE FILE LIST # of files that will be aggregted def create_file_list_fct(subjects_list, in_data_name, data_lookup_dict, template_lookup_dict): file_list = [] for s in subjects_list: file_list.append(data_lookup_dict[in_data_name]['path_str'].format(subject_id=s)) if 'matrix_name' in data_lookup_dict[in_data_name].keys(): matrix_name = data_lookup_dict[in_data_name]['matrix_name'] else: matrix_name = None if 'parcellation_path' in data_lookup_dict[in_data_name].keys(): parcellation_path = data_lookup_dict[in_data_name]['parcellation_path'] else: parcellation_path = None if 'fwhm' in data_lookup_dict[in_data_name].keys(): fwhm = data_lookup_dict[in_data_name]['fwhm'] if fwhm == 0: fwhm = None else: fwhm = None if 'mask_name' in data_lookup_dict[in_data_name].keys(): mask_path = template_lookup_dict[data_lookup_dict[in_data_name]['mask_name']] else: mask_path = None if 'use_diagonal' in data_lookup_dict[in_data_name].keys(): use_diagonal = data_lookup_dict[in_data_name]['use_diagonal'] else: use_diagonal = False if 'use_fishers_z' in data_lookup_dict[in_data_name].keys(): use_fishers_z = data_lookup_dict[in_data_name]['use_fishers_z'] else: use_fishers_z = False if 'df_col_names' in data_lookup_dict[in_data_name].keys(): df_col_names = data_lookup_dict[in_data_name]['df_col_names'] else: df_col_names = None return file_list, matrix_name, parcellation_path, fwhm, mask_path, use_diagonal, use_fishers_z, df_col_names create_file_list = Node(util.Function(input_names=['subjects_list', 'in_data_name', 'data_lookup_dict', 'template_lookup_dict', ], output_names=['file_list', 'matrix_name', 'parcellation_path', 'fwhm', 'mask_path', 'use_diagonal', 'use_fishers_z', 'df_col_names'], function=create_file_list_fct), name='create_file_list') wf.connect(create_df, 'full_subjects_list', create_file_list, 'subjects_list') wf.connect(in_data_name_infosource, 'in_data_name', create_file_list, 'in_data_name') create_file_list.inputs.data_lookup_dict = data_lookup_dict create_file_list.inputs.template_lookup_dict = template_lookup_dict ############################################################################################################### # VECTORIZE AND AGGREGATE SUBJECTS # stack single subject np arrays vertically vectorize_aggregate_subjects = Node(util.Function(input_names=['in_data_file_list', 'mask_file', 'matrix_name', 'parcellation_path', 'fwhm', 'use_diagonal', 'use_fishers_z', 'df_file', 'df_col_names'], output_names=['vectorized_aggregated_file', 'unimodal_backprojection_info_file'], function=vectorize_and_aggregate), name='vectorize_aggregate_subjects') wf.connect(create_file_list, 'file_list', vectorize_aggregate_subjects, 'in_data_file_list') wf.connect(create_file_list, 'mask_path', vectorize_aggregate_subjects, 'mask_file') wf.connect(create_file_list, 'matrix_name', vectorize_aggregate_subjects, 'matrix_name') wf.connect(create_file_list, 'parcellation_path', vectorize_aggregate_subjects, 'parcellation_path') wf.connect(create_file_list, 'fwhm', vectorize_aggregate_subjects, 'fwhm') wf.connect(create_file_list, 'use_diagonal', vectorize_aggregate_subjects, 'use_diagonal') wf.connect(create_file_list, 'use_fishers_z', vectorize_aggregate_subjects, 'use_fishers_z') wf.connect(create_df, 'df_all_subjects_pickle_file', vectorize_aggregate_subjects, 'df_file') wf.connect(create_file_list, 'df_col_names', vectorize_aggregate_subjects, 'df_col_names') wf.connect(create_df, 'df_all_subjects_pickle_file', ds_X, 'df_all_subjects_pickle_file') wf.connect(vectorize_aggregate_subjects, 'vectorized_aggregated_file', ds_X, 'X_file') wf.connect(vectorize_aggregate_subjects, 'unimodal_backprojection_info_file', ds_X, 'unimodal_backprojection_info_file') ##################################### # RUN WF ##################################### wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def downsampel_surfs(subject_id, working_dir, freesurfer_dir, ds_dir, plugin_name, use_n_procs): ''' Workflow resamples e.g. native thickness maps to fsaverage5 space ''' ##################################### # GENERAL SETTINGS ##################################### fsl.FSLCommand.set_default_output_type('NIFTI_GZ') fs.FSCommand.set_default_subjects_dir(freesurfer_dir) wf = Workflow(name='freesurfer_downsample') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True, 'remove_unnecessary_outputs': True, 'job_finished_timeout': 15}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash') ds = Node(nio.DataSink(base_directory=ds_dir), name='ds') ds.inputs.parameterization = False ##################### # ITERATORS ##################### # PARCELLATION ITERATOR infosource = Node(util.IdentityInterface(fields=['hemi', 'surf_measure', 'fwhm', 'target']), name='infosource') infosource.iterables = [('hemi', ['lh', 'rh']), ('surf_measure', ['thickness', 'area']), ('fwhm', [0, 5, 10, 20]), ('target', ['fsaverage3', 'fsaverage4', 'fsaverage5']), ] downsample = Node(fs.model.MRISPreproc(), name='downsample') downsample.inputs.subjects = [subject_id] wf.connect(infosource, 'target', downsample, 'target') wf.connect(infosource, 'hemi', downsample, 'hemi') wf.connect(infosource, 'surf_measure', downsample, 'surf_measure') wf.connect(infosource, 'fwhm', downsample, 'fwhm_source') rename = Node(util.Rename(format_string='%(hemi)s.%(surf_measure)s.%(target)s.%(fwhm)smm'), name='rename') rename.inputs.keep_ext = True wf.connect(infosource, 'target', rename, 'target') wf.connect(infosource, 'hemi', rename, 'hemi') wf.connect(infosource, 'surf_measure', rename, 'surf_measure') wf.connect(infosource, 'fwhm', rename, 'fwhm') wf.connect(downsample, 'out_file', rename, 'in_file') wf.connect(rename, 'out_file', ds, 'surfs.@surfs') # wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def collect_3d_metrics_run_glm_meanRegression(cfg): import os from nipype import config from nipype.pipeline.engine import Node, Workflow, MapNode, JoinNode import nipype.interfaces.utility as util import nipype.interfaces.io as nio import nipype.interfaces.fsl as fsl import nipype.interfaces.freesurfer as freesurfer # INPUT PARAMETERS metrics_data_dir = cfg['metrics_data_dir'] metrics_data_suffix = cfg['metrics_data_suffix'] metric_name = cfg['metric_name'] demos_df = cfg['demos_df'] qc_df = cfg['qc_df'] working_dir = cfg['working_dir'] ds_dir = cfg['ds_dir'] template_dir = cfg['template_dir'] subjects_list = cfg['subjects_list'] use_n_procs = cfg['use_n_procs'] plugin_name = cfg['plugin_name'] ##################################### # GENERAL SETTINGS ##################################### fsl.FSLCommand.set_default_output_type('NIFTI_GZ') wf = Workflow(name='LeiCA_collect_metrics') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True, 'remove_unnecessary_outputs': True, 'job_finished_timeout': 120}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash') ds = Node(nio.DataSink(base_directory=ds_dir), name='ds') # get atlas data templates_atlases = {'GM_mask_MNI_2mm': 'SPM_GM/SPM_GM_mask_2mm.nii.gz', 'GM_mask_MNI_3mm': 'SPM_GM/SPM_GM_mask_3mm.nii.gz', 'brain_mask_MNI_3mm': 'cpac_image_resources/MNI_3mm/MNI152_T1_3mm_brain_mask.nii.gz', 'brain_template_MNI_3mm': 'cpac_image_resources/MNI_3mm/MNI152_T1_3mm.nii.gz' } selectfiles_anat_templates = Node(nio.SelectFiles(templates_atlases, base_directory=template_dir), name="selectfiles_anat_templates") # EXPORT SUBJECT LIST def export_subjects_list_fct(subjects_list): import os out_file = os.path.join(os.getcwd(), 'subject_list.txt') file = open(out_file, 'w') for subject in subjects_list: file.write('%s\n'%subject) file.close() return out_file # RESTRICT SUBJECTS LIST TO ADULTS def get_subjects_list_adults_fct(df_path, df_qc_path, subjects_list): ''' excludes kids and subjects with missing sex or age ''' import pandas as pd import numpy as np df = pd.read_pickle(df_path) df_qc = pd.read_pickle(df_qc_path) df = pd.merge(df, df_qc, left_index=True, right_index=True) pd.to_pickle(df, 'testdf.pkl') df['subject_id'] = df.subject_id_x # fixme exclude subjects with mean_FD>.1 subjects_list_exclude = df[(df.age<18) | (df.mean_FD_Power>.1)].index subjects_list_adults = subjects_list for exclude_subject in subjects_list_exclude: if exclude_subject in subjects_list_adults: subjects_list_adults.remove(exclude_subject) missing_info = df[(df.age==999) | ((np.logical_or(df.sex=='M', df.sex=='F'))==False)].index for missing in missing_info: if missing in subjects_list_adults: subjects_list_adults.remove(missing) # remove subject from subject_list_adults for which no entry exists in df for subject in subjects_list_adults: if not(subject in df.index): subjects_list_adults.remove(subject) return subjects_list_adults get_subjects_list_adults = Node(util.Function(input_names=['df_path', 'df_qc_path', 'subjects_list'], output_names=['subjects_list_adults'], function=get_subjects_list_adults_fct), name='get_subjects_list_adults') get_subjects_list_adults.inputs.df_path = demos_df get_subjects_list_adults.inputs.df_qc_path = qc_df get_subjects_list_adults.inputs.subjects_list = subjects_list export_subjects_list = Node(util.Function(input_names=['subjects_list'], output_names=['out_file'], function=export_subjects_list_fct), name='export_subjects_list') wf.connect(get_subjects_list_adults, 'subjects_list_adults', export_subjects_list,'subjects_list') wf.connect(export_subjects_list, 'out_file', ds,'@subjects_list') # BUILD FILE LIST FOR MERGER def build_file_list_fct(prefix, subjects_list, suffix): import os out_file_list = [] for subject in subjects_list: out_file_list.append(os.path.join(prefix, subject, suffix)) return out_file_list build_file_list = Node(util.Function(input_names=['prefix', 'subjects_list', 'suffix'], output_names=['out_file_list'], function=build_file_list_fct), name='build_file_list') build_file_list.inputs.prefix = metrics_data_dir wf.connect(get_subjects_list_adults, 'subjects_list_adults', build_file_list,'subjects_list') build_file_list.inputs.suffix = metrics_data_suffix # MERGE FILES merge = Node(fsl.Merge(dimension='t'), name='merge') wf.connect(build_file_list, 'out_file_list', merge, 'in_files') merge.inputs.merged_file = metric_name + '_merge.nii.gz' wf.connect(merge, 'merged_file', ds,'@merge') # GET MEAN VALUE WITHIN MASK FOR REGRESSION get_mean_values = Node(fsl.ImageStats(), name='get_mean_values') get_mean_values.inputs.op_string='-k %s -m' get_mean_values.inputs.split_4d=True wf.connect(merge, 'merged_file', get_mean_values, 'in_file') wf.connect(selectfiles_anat_templates, 'brain_mask_MNI_3mm', get_mean_values, 'mask_file') # CALC MEAN mean = Node(fsl.MeanImage(), name='mean') mean.inputs.out_file = metric_name + '_mean.nii.gz' wf.connect(merge, 'merged_file', mean, 'in_file') wf.connect(mean, 'out_file', ds,'@mean') # CREATE DESIGN FILES def create_design_files_fct(df_demographics_path, df_qc_path, subjects_list, mean_values): ''' df_path: df should have columns sex ('M', 'F') & age function .restricts df to subjects_list .creates dummy sex vars, age**2, contrasts .writes mat & con files ''' import pandas as pd import numpy as np import os df = pd.read_pickle(df_demographics_path) df_qc = pd.read_pickle(df_qc_path) df = pd.merge(df, df_qc, left_index=True, right_index=True) df['dummy_m'] = (df.sex == 'M').astype('int') df['dummy_f'] = (df.sex == 'F').astype('int') df['age2'] = df.age**2 df_use = df.loc[subjects_list] df_use['mean_values'] = mean_values #fixme mat = df_use[['dummy_m', 'dummy_f', 'age','mean_FD_Power']].values mat_str = [ '/NumWaves %s'%str(mat.shape[1]), '/NumPoints %s'%str(mat.shape[0]), '/Matrix' ] n_cons = 6 cons_str = [ '/ContrastName1 pos_age', '/ContrastName2 neg_age', '/ContrastName3 m>f', '/ContrastName4 f>m', '/ContrastName5 pos_mean_FD_Power', '/ContrastName6 neg_mean_FD_Power', '/NumWaves %s'%str(mat.shape[1]), '/NumContrasts %s'%str(n_cons), '', '/Matrix', '0 0 1 0', '0 0 -1 0', '1 -1 0 0', '-1 1 0 0', '0 0 0 1', '0 0 0 -1' ] # mat = df_use[['dummy_m', 'dummy_f', 'age', 'age2','mean_FD_Power']].values # # mat_str = [ # '/NumWaves %s'%str(mat.shape[1]), # '/NumPoints %s'%str(mat.shape[0]), # '/Matrix' # ] # # n_cons = 10 # cons_str = [ # '/ContrastName1 pos_age', # '/ContrastName2 pos_age2', # '/ContrastName3 pos_age+age2', # '/ContrastName4 neg_age', # '/ContrastName5 neg_age2', # '/ContrastName6 neg_age+age2', # '/ContrastName7 m>f', # '/ContrastName8 f>m', # '/ContrastName9 pos_mean_FD_Power', # '/ContrastName10 neg_mean_FD_Power', # '/NumWaves %s'%str(mat.shape[1]), # '/NumContrasts %s'%str(n_cons), # '', # '/Matrix', # '0 0 1 0 0', # '0 0 0 1 0', # '0 0 .5 .5 0', # '0 0 -1 0 0', # '0 0 0 -1 0', # '0 0 -.5 -.5 0', # '1 -1 0 0 0', # '1 1 0 0 0', # '0 0 0 0 1', # '0 0 0 0 -1' # ] # mat_file = os.path.join(os.getcwd(), 'design.mat') mat_file_numerical = os.path.join(os.getcwd(), 'design_mat_num.txt') con_file = os.path.join(os.getcwd(), 'design.con') df_used_file = os.path.join(os.getcwd(), 'df_used.csv') # mat_str_out = '' # for line in mat_str: # mat_str_out = mat_str_out[:] + line + '\n' np.savetxt(mat_file_numerical, mat) num_file = open(mat_file_numerical, 'r') nums_str = num_file.read() num_file.close() out_file = open(mat_file, 'w') for line in mat_str: out_file.write('%s\n' % line) out_file.write(nums_str) out_file.close() out_file = open(con_file, 'w') for line in cons_str: out_file.write('%s\n' % line) out_file.close() df_use.to_csv(df_used_file) return mat_file, con_file, df_used_file, n_cons create_design_files = Node(util.Function(input_names=['df_demographics_path', 'df_qc_path', 'subjects_list', 'mean_values'], output_names=['mat_file', 'con_file', 'df_used_file', 'n_cons'], function=create_design_files_fct), name='create_design_files') create_design_files.inputs.df_demographics_path = demos_df create_design_files.inputs.df_qc_path = qc_df wf.connect(get_subjects_list_adults, 'subjects_list_adults', create_design_files,'subjects_list') wf.connect(get_mean_values, 'out_stat', create_design_files,'mean_values') wf.connect(create_design_files,'df_used_file', ds,'glm.@df_used') smooth = Node(fsl.utils.Smooth(fwhm=4), name='smooth') wf.connect(merge, 'merged_file', smooth, 'in_file') def run_randomise_fct(data_file, mat_file, con_file, mask_file): import os out_dir = os.path.join(os.getcwd(), 'glm') os.mkdir(out_dir) cmd_str = 'randomise -i %s -o glm/glm -d %s -t %s -m %s -n 500 -D -T' %(data_file,mat_file, con_file, mask_file) file = open('command.txt', 'w') file.write(cmd_str) file.close() os.system(cmd_str) return out_dir run_randomise = Node(util.Function(input_names=['data_file', 'mat_file', 'con_file', 'mask_file'], output_names=['out_dir'], function=run_randomise_fct), name='run_randomise') #fixme #wf.connect(merge, 'merged_file', run_randomise, 'data_file') wf.connect(smooth, 'smoothed_file', run_randomise, 'data_file') wf.connect(create_design_files, 'mat_file', run_randomise, 'mat_file') wf.connect(create_design_files, 'con_file', run_randomise, 'con_file') #wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', run_randomise, 'mask_file') wf.connect(selectfiles_anat_templates, 'brain_mask_MNI_3mm', run_randomise, 'mask_file') def create_renders_fct(randomise_dir, n_cons, background_img): import os thresh = .95 out_files_list = [] corr_list = ['glm_tfce_corrp_tstat', 'glm_tfce_p_tstat'] for corr in corr_list: for con in range(1,n_cons+1): output_root = corr + str(con) in_file = os.path.join(randomise_dir, output_root + '.nii.gz') out_file = os.path.join(os.getcwd(), 'rendered_' + output_root + '.png') out_files_list.append(out_file) cmd_str = 'easythresh %s %s %s %s' % (in_file, str(thresh), background_img, output_root) file = open('command.txt', 'w') file.write(cmd_str) file.close() os.system(cmd_str) return out_files_list create_renders = Node(util.Function(input_names=['randomise_dir', 'n_cons', 'background_img'], output_names=['out_files_list'], function=create_renders_fct), name='create_renders') wf.connect(run_randomise, 'out_dir', create_renders, 'randomise_dir') wf.connect(create_design_files, 'n_cons', create_renders, 'n_cons') wf.connect(selectfiles_anat_templates, 'brain_template_MNI_3mm', create_renders, 'background_img') wf.connect(create_renders, 'out_files_list', ds, 'glm.@renders') wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def create_workflow(opts): import logging from nipype import config as ncfg from fmriprep.utils import make_folder from fmriprep.viz.reports import run_reports from fmriprep.workflows.base import base_workflow_enumerator settings = { 'bids_root': op.abspath(opts.bids_dir), 'write_graph': opts.write_graph, 'nthreads': opts.nthreads, 'mem_mb': opts.mem_mb, 'debug': opts.debug, 'ants_nthreads': opts.ants_nthreads, 'skull_strip_ants': opts.skull_strip_ants, 'output_dir': op.abspath(opts.output_dir), 'work_dir': op.abspath(opts.work_dir), 'workflow_type': opts.workflow_type, 'skip_native': opts.skip_native } # set up logger logger = logging.getLogger('cli') if opts.debug: settings['ants_t1-mni_settings'] = 't1-mni_registration_test' logger.setLevel(logging.DEBUG) log_dir = op.join(settings['output_dir'], 'log') derivatives = op.join(settings['output_dir'], 'derivatives') # Check and create output and working directories # Using make_folder to prevent https://github.com/poldracklab/mriqc/issues/111 make_folder(settings['output_dir']) make_folder(settings['work_dir']) make_folder(derivatives) make_folder(log_dir) logger.addHandler(logging.FileHandler(op.join(log_dir, 'run_workflow'))) # Set nipype config ncfg.update_config({ 'logging': {'log_directory': log_dir, 'log_to_file': True}, 'execution': {'crashdump_dir': log_dir, 'remove_unnecessary_outputs': False} }) # nipype plugin configuration plugin_settings = {'plugin': 'Linear'} if opts.use_plugin is not None: from yaml import load as loadyml with open(opts.use_plugin) as f: plugin_settings = loadyml(f) else: # Setup multiprocessing if settings['nthreads'] == 0: settings['nthreads'] = cpu_count() if settings['nthreads'] > 1: plugin_settings['plugin'] = 'MultiProc' plugin_settings['plugin_args'] = {'n_procs': settings['nthreads']} if settings['mem_mb']: plugin_settings['plugin_args']['memory_gb'] = settings['mem_mb']/1024 if settings['ants_nthreads'] == 0: settings['ants_nthreads'] = cpu_count() # Determine subjects to be processed subject_list = opts.participant_label if subject_list is None or not subject_list: subject_list = [op.basename(subdir)[4:] for subdir in glob.glob( op.join(settings['bids_root'], 'sub-*'))] logger.info('Subject list: %s', ', '.join(subject_list)) # Build main workflow and run preproc_wf = base_workflow_enumerator(subject_list, task_id=opts.task_id, settings=settings) preproc_wf.base_dir = settings['work_dir'] preproc_wf.run(**plugin_settings) if opts.write_graph: preproc_wf.write_graph(graph2use="colored", format='svg', simple_form=True) run_reports(settings['output_dir'])
def main(args): subjects, master_config = args import os import sys import traceback # Set universal pipeline options from nipype import config config.update_config(master_config) assert config.get('execution', 'plugin') == master_config['execution']['plugin'] import nipype.pipeline.engine as pe import nipype.interfaces.io as nio from nipype.interfaces.utility import IdentityInterface, Function import nipype.interfaces.ants as ants from template import MergeByExtendListElements, xml_filename from PipeLineFunctionHelpers import mapPosteriorList from atlasNode import GetAtlasNode, MakeNewAtlasTemplate from utilities.misc import GenerateSubjectOutputPattern as outputPattern from utilities.distributed import modify_qsub_args template = pe.Workflow(name='SubjectAtlas_Template') template.base_dir = master_config['logging']['log_directory'] if 'previouscache' in master_config: # Running off previous baseline experiment BAtlas = GetAtlasNode(master_config['previouscache'], 'BAtlas') else: # Running after previous baseline experiment BAtlas = GetAtlasNode(os.path.dirname(master_config['atlascache']), 'BAtlas') inputspec = pe.Node(interface=IdentityInterface(fields=['subject']), name='inputspec') inputspec.iterables = ('subject', subjects) baselineDG = pe.Node(nio.DataGrabber(infields=['subject'], outfields=['t1_average', 't2_average', 'pd_average', 'fl_average', 'outputLabels', 'posteriorImages']), name='Baseline_DG') if 'previousresult' in master_config: baselineDG.inputs.base_directory = master_config['previousresult'] else: baselineDG.inputs.base_directory = master_config['resultdir'] baselineDG.inputs.sort_filelist = True baselineDG.inputs.raise_on_empty = False baselineDG.inputs.template = '*/%s/*/Baseline/%s.nii.gz' baselineDG.inputs.template_args['t1_average'] = [['subject', 't1_average_BRAINSABC']] baselineDG.inputs.template_args['t2_average'] = [['subject', 't2_average_BRAINSABC']] baselineDG.inputs.template_args['pd_average'] = [['subject', 'pd_average_BRAINSABC']] baselineDG.inputs.template_args['fl_average'] = [['subject', 'fl_average_BRAINSABC']] baselineDG.inputs.template_args['outputLabels'] = [['subject', 'brain_label_seg']] baselineDG.inputs.field_template = {'posteriorImages':'*/%s/*/TissueClassify/POSTERIOR_%s.nii.gz'} posterior_files = ['AIR', 'BASAL', 'CRBLGM', 'CRBLWM', 'CSF', 'GLOBUS', 'HIPPOCAMPUS', 'NOTCSF', 'NOTGM', 'NOTVB', 'NOTWM', 'SURFGM', 'THALAMUS', 'VB', 'WM'] baselineDG.inputs.template_args['posteriorImages'] = [['subject', posterior_files]] MergeByExtendListElementsNode = pe.Node(Function(function=MergeByExtendListElements, input_names=['t1s', 't2s', 'pds', 'fls', 'labels', 'posteriors'], output_names=['ListOfImagesDictionaries', 'registrationImageTypes', 'interpolationMapping']), run_without_submitting=True, name="99_MergeByExtendListElements") from PipeLineFunctionHelpers import WrapPosteriorImagesFromDictionaryFunction as wrapfunc template.connect([(inputspec, baselineDG, [('subject', 'subject')]), (baselineDG, MergeByExtendListElementsNode, [('t1_average', 't1s'), ('t2_average', 't2s'), ('pd_average', 'pds'), ('fl_average', 'fls'), ('outputLabels', 'labels'), (('posteriorImages', wrapfunc), 'posteriors')]) ]) myInitAvgWF = pe.Node(interface=ants.AverageImages(), name='Atlas_antsSimpleAverage') # was 'Phase1_antsSimpleAverage' myInitAvgWF.inputs.dimension = 3 myInitAvgWF.inputs.normalize = True template.connect(baselineDG, 't1_average', myInitAvgWF, "images") #################################################################################################### # TEMPLATE_BUILD_RUN_MODE = 'MULTI_IMAGE' # if numSessions == 1: # TEMPLATE_BUILD_RUN_MODE = 'SINGLE_IMAGE' #################################################################################################### from BAWantsRegistrationBuildTemplate import BAWantsRegistrationTemplateBuildSingleIterationWF as registrationWF buildTemplateIteration1 = registrationWF('iteration01') # buildTemplateIteration2 = buildTemplateIteration1.clone(name='buildTemplateIteration2') buildTemplateIteration2 = registrationWF('Iteration02') MakeNewAtlasTemplateNode = pe.Node(interface=Function(function=MakeNewAtlasTemplate, input_names=['t1_image', 'deformed_list', 'AtlasTemplate', 'outDefinition'], output_names=['outAtlasFullPath', 'clean_deformed_list']), # This is a lot of work, so submit it run_without_submitting=True, run_without_submitting=True, # HACK: THIS NODE REALLY SHOULD RUN ON THE CLUSTER! name='99_MakeNewAtlasTemplate') if master_config['execution']['plugin'] == 'SGE': # for some nodes, the qsub call needs to be modified on the cluster MakeNewAtlasTemplateNode.plugin_args = {'template': master_config['plugin_args']['template'], 'qsub_args': modify_qsub_args(master_config['queue'], '1000M', 1, 1), 'overwrite': True} for bt in [buildTemplateIteration1, buildTemplateIteration2]: ################################################## # *** Hans, is this TODO already addressed? *** # # ----> # TODO: Change these parameters <---- # ################################################## BeginANTS = bt.get_node("BeginANTS") BeginANTS.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], '9000M', 4, hard=False)} wimtdeformed = bt.get_node("wimtdeformed") wimtdeformed.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], '2000M', 1, 2)} AvgAffineTransform = bt.get_node("AvgAffineTransform") AvgAffineTransform.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], '2000M', 1)} wimtPassivedeformed = bt.get_node("wimtPassivedeformed") wimtPassivedeformed.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], '2000M', 1, 2)} template.connect([(myInitAvgWF, buildTemplateIteration1, [('output_average_image', 'inputspec.fixed_image')]), (MergeByExtendListElementsNode, buildTemplateIteration1, [('ListOfImagesDictionaries', 'inputspec.ListOfImagesDictionaries'), ('registrationImageTypes', 'inputspec.registrationImageTypes'), ('interpolationMapping','inputspec.interpolationMapping')]), (buildTemplateIteration1, buildTemplateIteration2, [('outputspec.template', 'inputspec.fixed_image')]), (MergeByExtendListElementsNode, buildTemplateIteration2, [('ListOfImagesDictionaries', 'inputspec.ListOfImagesDictionaries'), ('registrationImageTypes','inputspec.registrationImageTypes'), ('interpolationMapping', 'inputspec.interpolationMapping')]), (inputspec, MakeNewAtlasTemplateNode, [(('subject', xml_filename), 'outDefinition')]), (BAtlas, MakeNewAtlasTemplateNode, [('ExtendedAtlasDefinition_xml_in', 'AtlasTemplate')]), (buildTemplateIteration2, MakeNewAtlasTemplateNode, [('outputspec.template', 't1_image'), ('outputspec.passive_deformed_templates', 'deformed_list')]), ]) # Create DataSinks Atlas_DataSink = pe.Node(nio.DataSink(), name="Atlas_DS") Atlas_DataSink.overwrite = master_config['ds_overwrite'] Atlas_DataSink.inputs.base_directory = master_config['resultdir'] Subject_DataSink = pe.Node(nio.DataSink(), name="Subject_DS") Subject_DataSink.overwrite = master_config['ds_overwrite'] Subject_DataSink.inputs.base_directory = master_config['resultdir'] template.connect([(inputspec, Atlas_DataSink, [('subject', 'container')]), (buildTemplateIteration1, Atlas_DataSink, [('outputspec.template', 'Atlas.iteration1')]), # Unnecessary (MakeNewAtlasTemplateNode, Atlas_DataSink, [('outAtlasFullPath', 'Atlas.definitions')]), (BAtlas, Atlas_DataSink, [('template_landmarks_50Lmks_fcsv', 'Atlas.20111119_BCD.@fcsv'), ('template_weights_50Lmks_wts', 'Atlas.20111119_BCD.@wts'), ('LLSModel_50Lmks_hdf5', 'Atlas.20111119_BCD.@hdf5'), ('T1_50Lmks_mdl', 'Atlas.20111119_BCD.@mdl')]), (inputspec, Subject_DataSink, [(('subject', outputPattern), 'regexp_substitutions')]), (buildTemplateIteration2, Subject_DataSink, [('outputspec.template', 'ANTSTemplate.@template')]), (MakeNewAtlasTemplateNode, Subject_DataSink, [('clean_deformed_list', 'ANTSTemplate.@passive_deformed_templates')]), ]) from utils import run_workflow, print_workflow if False: print_workflow(template, plugin=master_config['execution']['plugin'], dotfilename='template') return run_workflow(template, plugin=master_config['execution']['plugin'], plugin_args=master_config['plugin_args'])
def process(self): # Process time now = datetime.datetime.now().strftime("%Y%m%d_%H%M") # Initialization if os.path.exists(os.path.join(self.base_directory,"LOG","pypeline.log")): os.unlink(os.path.join(self.base_directory,"LOG","pypeline.log")) config.update_config({'logging': {'log_directory': os.path.join(self.base_directory,"LOG"), 'log_to_file': True}, 'execution': {'remove_unnecessary_outputs': False} }) logging.update_logging(config) iflogger = logging.getLogger('interface') # Data import datasource = pe.Node(interface=nio.DataGrabber(outfields = ['fMRI','T1','T2']), name='datasource') datasource.inputs.base_directory = os.path.join(self.base_directory,'NIFTI') datasource.inputs.template = '*' datasource.inputs.raise_on_empty = False datasource.inputs.field_template = dict(fMRI='fMRI.nii.gz',T1='T1.nii.gz',T2='T2.nii.gz') datasource.inputs.sort_filelist=False # Data sinker for output sinker = pe.Node(nio.DataSink(), name="fMRI_sinker") sinker.inputs.base_directory = os.path.join(self.base_directory, "RESULTS") # Clear previous outputs self.clear_stages_outputs() # Create common_flow common_flow = self.create_common_flow() # Create fMRI flow fMRI_flow = pe.Workflow(name='fMRI_pipeline') fMRI_inputnode = pe.Node(interface=util.IdentityInterface(fields=["fMRI","T1","T2","subjects_dir","subject_id","wm_mask_file","roi_volumes","wm_eroded","brain_eroded","csf_eroded","parcellation_scheme","atlas_info"]),name="inputnode") fMRI_outputnode = pe.Node(interface=util.IdentityInterface(fields=["connectivity_matrices"]),name="outputnode") fMRI_flow.add_nodes([fMRI_inputnode,fMRI_outputnode]) if self.stages['Preprocessing'].enabled: preproc_flow = self.create_stage_flow("Preprocessing") fMRI_flow.connect([ (fMRI_inputnode,preproc_flow,[("fMRI","inputnode.functional")]), ]) if self.stages['Registration'].enabled: reg_flow = self.create_stage_flow("Registration") fMRI_flow.connect([ (fMRI_inputnode,reg_flow,[('T1','inputnode.T1')]),(fMRI_inputnode,reg_flow,[('T2','inputnode.T2')]), (preproc_flow,reg_flow, [('outputnode.mean_vol','inputnode.target')]), (fMRI_inputnode,reg_flow, [('wm_mask_file','inputnode.wm_mask'),('roi_volumes','inputnode.roi_volumes'), ('wm_eroded','inputnode.eroded_wm')]) ]) if self.stages['Functional'].config.global_nuisance: fMRI_flow.connect([ (fMRI_inputnode,reg_flow,[('brain_eroded','inputnode.eroded_brain')]) ]) if self.stages['Functional'].config.csf: fMRI_flow.connect([ (fMRI_inputnode,reg_flow,[('csf_eroded','inputnode.eroded_csf')]) ]) if self.stages['Registration'].config.registration_mode == "BBregister (FS)": fMRI_flow.connect([ (fMRI_inputnode,reg_flow, [('subjects_dir','inputnode.subjects_dir'), ('subject_id','inputnode.subject_id')]), ]) if self.stages['Functional'].enabled: func_flow = self.create_stage_flow("Functional") fMRI_flow.connect([ (preproc_flow,func_flow, [('outputnode.functional_preproc','inputnode.preproc_file')]), (reg_flow,func_flow, [('outputnode.wm_mask_registered','inputnode.registered_wm'),('outputnode.roi_volumes_registered','inputnode.registered_roi_volumes'), ('outputnode.eroded_wm_registered','inputnode.eroded_wm'),('outputnode.eroded_csf_registered','inputnode.eroded_csf'), ('outputnode.eroded_brain_registered','inputnode.eroded_brain')]) ]) if self.stages['Functional'].config.scrubbing or self.stages['Functional'].config.motion: fMRI_flow.connect([ (preproc_flow,func_flow,[("outputnode.par_file","inputnode.motion_par_file")]) ]) if self.stages['Connectome'].enabled: con_flow = self.create_stage_flow("Connectome") fMRI_flow.connect([ (fMRI_inputnode,con_flow, [('parcellation_scheme','inputnode.parcellation_scheme')]), (func_flow,con_flow, [('outputnode.func_file','inputnode.func_file'),("outputnode.FD","inputnode.FD"), ("outputnode.DVARS","inputnode.DVARS")]), (reg_flow,con_flow,[("outputnode.roi_volumes_registered","inputnode.roi_volumes_registered")]), (con_flow,fMRI_outputnode,[("outputnode.connectivity_matrices","connectivity_matrices")]) ]) if self.stages['Parcellation'].config.parcellation_scheme == "Custom": fMRI_flow.connect([(fMRI_inputnode,con_flow, [('atlas_info','inputnode.atlas_info')])]) # Create NIPYPE flow flow = pe.Workflow(name='NIPYPE', base_dir=os.path.join(self.base_directory)) flow.connect([ (datasource,common_flow,[("T1","inputnode.T1")]), (datasource,fMRI_flow,[("fMRI","inputnode.fMRI"),("T1","inputnode.T1"),("T2","inputnode.T2")]), (common_flow,fMRI_flow,[("outputnode.subjects_dir","inputnode.subjects_dir"), ("outputnode.subject_id","inputnode.subject_id"), ("outputnode.wm_mask_file","inputnode.wm_mask_file"), ("outputnode.roi_volumes","inputnode.roi_volumes"), ("outputnode.wm_eroded","inputnode.wm_eroded"), ("outputnode.brain_eroded","inputnode.brain_eroded"), ("outputnode.csf_eroded","inputnode.csf_eroded"), ("outputnode.parcellation_scheme","inputnode.parcellation_scheme"), ("outputnode.atlas_info","inputnode.atlas_info")]), (fMRI_flow,sinker,[("outputnode.connectivity_matrices","fMRI.%s.connectivity_matrices"%now)]) ]) # Process pipeline iflogger.info("**** Processing ****") if(self.number_of_cores != 1): flow.run(plugin='MultiProc', plugin_args={'n_procs' : self.number_of_cores}) else: flow.run() self.fill_stages_outputs() # Clean undesired folders/files rm_file_list = ['rh.EC_average','lh.EC_average','fsaverage'] for file_to_rm in rm_file_list: if os.path.exists(os.path.join(self.base_directory,file_to_rm)): os.remove(os.path.join(self.base_directory,file_to_rm)) # copy .ini and log file outdir = os.path.join(self.base_directory,"RESULTS",'fMRI',now) if not os.path.exists(outdir): os.makedirs(outdir) shutil.copy(self.config_file,outdir) shutil.copy(os.path.join(self.base_directory,'LOG','pypeline.log'),outdir) iflogger.info("**** Processing finished ****") return True,'Processing sucessful'
def init_mriqc(opts, retval): """Build the workflow enumerator""" from bids.grabbids import BIDSLayout from nipype import config as ncfg from nipype.pipeline.engine import Workflow from ..utils.bids import collect_bids_data from ..workflows.core import build_workflow retval['workflow'] = None retval['plugin_settings'] = None # Build settings dict bids_dir = Path(opts.bids_dir).expanduser() output_dir = Path(opts.output_dir).expanduser() # Number of processes n_procs = opts.n_procs or cpu_count() settings = { 'bids_dir': bids_dir.resolve(), 'output_dir': output_dir.resolve(), 'work_dir': opts.work_dir.expanduser().resolve(), 'write_graph': opts.write_graph, 'n_procs': n_procs, 'testing': opts.testing, 'hmc_afni': opts.hmc_afni, 'hmc_fsl': opts.hmc_fsl, 'fft_spikes_detector': opts.fft_spikes_detector, 'ants_nthreads': opts.ants_nthreads, 'ants_float': opts.ants_float, 'verbose_reports': opts.verbose_reports or opts.testing, 'float32': opts.float32, 'ica': opts.ica, 'no_sub': opts.no_sub, 'email': opts.email, 'fd_thres': opts.fd_thres, 'webapi_url': opts.webapi_url, 'webapi_port': opts.webapi_port, 'upload_strict': opts.upload_strict, } if opts.hmc_afni: settings['deoblique'] = opts.deoblique settings['despike'] = opts.despike settings['correct_slice_timing'] = opts.correct_slice_timing if opts.start_idx: settings['start_idx'] = opts.start_idx if opts. stop_idx: settings['stop_idx'] = opts.stop_idx if opts.ants_settings: settings['ants_settings'] = opts.ants_settings if opts.dsname: settings['dataset_name'] = opts.dsname log_dir = settings['output_dir'] / 'logs' # Create directories log_dir.mkdir(parents=True, exist_ok=True) settings['work_dir'].mkdir(parents=True, exist_ok=True) # Set nipype config ncfg.update_config({ 'logging': {'log_directory': str(log_dir), 'log_to_file': True}, 'execution': { 'crashdump_dir': str(log_dir), 'crashfile_format': 'txt', 'resource_monitor': opts.profile}, }) # Plugin configuration plugin_settings = {} if n_procs == 1: plugin_settings['plugin'] = 'Linear' if settings['ants_nthreads'] == 0: settings['ants_nthreads'] = 1 else: plugin_settings['plugin'] = 'MultiProc' plugin_settings['plugin_args'] = {'n_procs': n_procs} if opts.mem_gb: plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb if settings['ants_nthreads'] == 0: # always leave one extra thread for non ANTs work, # don't use more than 8 threads - the speed up is minimal settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8) # Overwrite options if --use-plugin provided if opts.use_plugin and opts.use_plugin.exists(): from yaml import load as loadyml with opts.use_plugin.open() as pfile: plugin_settings.update(loadyml(pfile)) # Process data types modalities = opts.modalities layout = BIDSLayout(str(settings['bids_dir']), exclude=['derivatives', 'sourcedata']) dataset = collect_bids_data( layout, participant_label=opts.participant_label, session=opts.session_id, run=opts.run_id, task=opts.task_id, bids_type=modalities, ) workflow = Workflow(name='workflow_enumerator') workflow.base_dir = settings['work_dir'] wf_list = [] subject_list = [] for mod in modalities: if dataset[mod]: wf_list.append(build_workflow(dataset[mod], mod, settings=settings)) subject_list += dataset[mod] retval['subject_list'] = subject_list if not wf_list: retval['return_code'] = 1 return retval workflow.add_nodes(wf_list) retval['plugin_settings'] = plugin_settings retval['workflow'] = workflow retval['return_code'] = 0 return retval
def calc_local_metrics(brain_mask, preprocessed_data_dir, subject_id, parcellations_dict, bp_freq_list, TR, selectfiles_templates, working_dir, ds_dir, use_n_procs, plugin_name): import os from nipype import config from nipype.pipeline.engine import Node, Workflow, MapNode import nipype.interfaces.utility as util import nipype.interfaces.io as nio import nipype.interfaces.fsl as fsl from nipype.interfaces.freesurfer.preprocess import MRIConvert import CPAC.alff.alff as cpac_alff import CPAC.reho.reho as cpac_reho import CPAC.utils.utils as cpac_utils import utils as calc_metrics_utils from motion import calculate_FD_P, calculate_FD_J ##################################### # GENERAL SETTINGS ##################################### fsl.FSLCommand.set_default_output_type('NIFTI_GZ') wf = Workflow(name='LeiCA_LIFE_metrics') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True, 'remove_unnecessary_outputs': True, 'job_finished_timeout': 15}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash') ds = Node(nio.DataSink(base_directory=ds_dir), name='ds') ds.inputs.regexp_substitutions = [('MNI_resampled_brain_mask_calc.nii.gz', 'falff.nii.gz'), ('residual_filtered_3dT.nii.gz', 'alff.nii.gz'), ('_parcellation_', ''), ('_bp_freqs_', 'bp_'), ] ##################### # ITERATORS ##################### # PARCELLATION ITERATOR parcellation_infosource = Node(util.IdentityInterface(fields=['parcellation']), name='parcellation_infosource') parcellation_infosource.iterables = ('parcellation', parcellations_dict.keys()) # BP FILTER ITERATOR bp_filter_infosource = Node(util.IdentityInterface(fields=['bp_freqs']), name='bp_filter_infosource') bp_filter_infosource.iterables = ('bp_freqs', bp_freq_list) selectfiles = Node(nio.SelectFiles(selectfiles_templates, base_directory=preprocessed_data_dir), name='selectfiles') selectfiles.inputs.subject_id = subject_id # ##################### # # FIX TR IN HEADER # ##################### # tr_msec = int(TR * 1000) # tr_str = '-tr %s' % tr_msec # # fixed_tr_bp = Node(MRIConvert(out_type='niigz', args=tr_str), name='fixed_tr_bp') # wf.connect(selectfiles, 'epi_MNI_bp', fixed_tr_bp, 'in_file') # # fixed_tr_fullspectrum = Node(MRIConvert(out_type='niigz', args=tr_str), name='fixed_tr_fullspectrum') # wf.connect(selectfiles, 'epi_MNI_fullspectrum', fixed_tr_fullspectrum, 'in_file') ##################### # calc FD ##################### FD_P = Node(util.Function(input_names=['in_file'], output_names=['FD_ts_file', 'mean_FD_file', 'max_FD_file'], function=calculate_FD_P), name='FD_P') wf.connect(selectfiles, 'moco_parms_file', FD_P, 'in_file') wf.connect(FD_P, 'FD_ts_file', ds, 'QC.@FD') wf.connect(FD_P, 'mean_FD_file', ds, 'QC.@mean_FD') wf.connect(FD_P, 'max_FD_file', ds, 'QC.@max_FD') FD_J = Node(util.Function(input_names=['in_file'], output_names=['FD_ts_file', 'mean_FD_file', 'max_FD_file'], function=calculate_FD_J), name='FD_J') wf.connect(selectfiles, 'jenkinson_file', FD_J, 'in_file') wf.connect(FD_J, 'FD_ts_file', ds, 'QC.@FD_J') wf.connect(FD_J, 'mean_FD_file', ds, 'QC.@mean_FD_J') wf.connect(FD_J, 'max_FD_file', ds, 'QC.@max_FD_J') wf.connect(selectfiles, 'rest2anat_cost_file', ds, 'QC.@cost_file') ##################### # CALCULATE METRICS ##################### # f/ALFF alff = cpac_alff.create_alff('alff') alff.inputs.hp_input.hp = 0.01 alff.inputs.lp_input.lp = 0.1 alff.inputs.inputspec.rest_mask = brain_mask #wf.connect(fixed_tr_fullspectrum, 'out_file', alff, 'inputspec.rest_res') wf.connect(selectfiles, 'epi_MNI_fullspectrum', alff, 'inputspec.rest_res') wf.connect(alff, 'outputspec.alff_img', ds, 'alff.@alff') wf.connect(alff, 'outputspec.falff_img', ds, 'alff.@falff') # f/ALFF_MNI Z-SCORE alff_z = cpac_utils.get_zscore(input_name='alff', wf_name='alff_z') alff_z.inputs.inputspec.mask_file = brain_mask wf.connect(alff, 'outputspec.alff_img', alff_z, 'inputspec.input_file') wf.connect(alff_z, 'outputspec.z_score_img', ds, 'alff_z.@alff') falff_z = cpac_utils.get_zscore(input_name='falff', wf_name='falff_z') falff_z.inputs.inputspec.mask_file = brain_mask wf.connect(alff, 'outputspec.falff_img', falff_z, 'inputspec.input_file') wf.connect(falff_z, 'outputspec.z_score_img', ds, 'alff_z.@falff') # REHO reho = cpac_reho.create_reho() reho.inputs.inputspec.cluster_size = 27 reho.inputs.inputspec.rest_mask = brain_mask #wf.connect(fixed_tr_bp, 'out_file', reho, 'inputspec.rest_res_filt') wf.connect(selectfiles, 'epi_MNI_BP', reho, 'inputspec.rest_res_filt') wf.connect(reho, 'outputspec.raw_reho_map', ds, 'reho.@reho') # VARIABILITY SCORES variability = Node(util.Function(input_names=['in_file'], output_names=['out_file'], function=calc_metrics_utils.calc_variability), name='variability') #wf.connect(fixed_tr_bp, 'out_file', variability, 'in_file') wf.connect(selectfiles, 'epi_MNI_BP', variability, 'in_file') wf.connect(variability, 'out_file', ds, 'variability.@SD') variability_z = cpac_utils.get_zscore(input_name='ts_std', wf_name='variability_z') variability_z.inputs.inputspec.mask_file = brain_mask wf.connect(variability, 'out_file', variability_z, 'inputspec.input_file') wf.connect(variability_z, 'outputspec.z_score_img', ds, 'variability_z.@variability_z') ############## ## CON MATS ############## ############## ## extract ts ############## parcellated_ts = Node( util.Function(input_names=['in_data', 'parcellation_name', 'parcellations_dict', 'bp_freqs', 'tr'], output_names=['parcellation_time_series', 'parcellation_time_series_file', 'masker_file'], function=calc_metrics_utils.extract_parcellation_time_series), name='parcellated_ts') parcellated_ts.inputs.parcellations_dict = parcellations_dict parcellated_ts.inputs.tr = TR #wf.connect(fixed_tr_fullspectrum, 'out_file', parcellated_ts, 'in_data') wf.connect(selectfiles, 'epi_MNI_fullspectrum', parcellated_ts, 'in_data') wf.connect(parcellation_infosource, 'parcellation', parcellated_ts, 'parcellation_name') wf.connect(bp_filter_infosource, 'bp_freqs', parcellated_ts, 'bp_freqs') ############## ## get conmat ############## con_mat = Node(util.Function(input_names=['in_data', 'extraction_method'], output_names=['matrix', 'matrix_file'], function=calc_metrics_utils.calculate_connectivity_matrix), name='con_mat') con_mat.inputs.extraction_method = 'correlation' wf.connect(parcellated_ts, 'parcellation_time_series', con_mat, 'in_data') ############## ## ds ############## wf.connect(parcellated_ts, 'parcellation_time_series_file', ds, 'con_mat.parcellated_time_series.@parc_ts') wf.connect(parcellated_ts, 'masker_file', ds, 'con_mat.parcellated_time_series.@masker') wf.connect(con_mat, 'matrix_file', ds, 'con_mat.matrix.@mat') wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name, plugin_args={'initial_specs': 'request_memory = 1500'}) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def main(): """Entry point""" from nipype import config as ncfg from nipype.pipeline.engine import Workflow from mriqc import DEFAULTS from mriqc.utils.bids import collect_bids_data from mriqc.workflows.core import build_workflow # from mriqc.reports.utils import check_reports parser = ArgumentParser(description='MRI Quality Control', formatter_class=RawTextHelpFormatter) parser.add_argument('-v', '--version', action='version', version='mriqc v{}'.format(__version__)) parser.add_argument('bids_dir', action='store', help='The directory with the input dataset ' 'formatted according to the BIDS standard.') parser.add_argument( 'output_dir', action='store', help='The directory where the output files ' 'should be stored. If you are running group level analysis ' 'this folder should be prepopulated with the results of the' 'participant level analysis.') parser.add_argument( 'analysis_level', action='store', nargs='+', help='Level of the analysis that will be performed. ' 'Multiple participant level analyses can be run independently ' '(in parallel) using the same output_dir.', choices=['participant', 'group']) parser.add_argument( '--participant_label', '--subject_list', '-S', action='store', help='The label(s) of the participant(s) that should be analyzed. ' 'The label corresponds to sub-<participant_label> from the ' 'BIDS spec (so it does not include "sub-"). If this parameter ' 'is not provided all subjects should be analyzed. Multiple ' 'participants can be specified with a space separated list.', nargs="*") g_input = parser.add_argument_group('mriqc specific inputs') g_input.add_argument('-m', '--modalities', action='store', nargs='*', choices=['T1w', 'bold', 'T2w'], default=['T1w', 'bold', 'T2w']) g_input.add_argument('-s', '--session-id', action='store') g_input.add_argument('-r', '--run-id', action='store') g_input.add_argument('--nthreads', action='store', type=int, help='number of threads') g_input.add_argument('--n_procs', action='store', default=0, type=int, help='number of threads') g_input.add_argument('--mem_gb', action='store', default=0, type=int, help='available total memory') g_input.add_argument('--write-graph', action='store_true', default=False, help='Write workflow graph.') g_input.add_argument('--dry-run', action='store_true', default=False, help='Do not run the workflow.') g_input.add_argument('--use-plugin', action='store', default=None, help='nipype plugin configuration file') g_input.add_argument('--testing', action='store_true', default=False, help='use testing settings for a minimal footprint') g_input.add_argument( '--hmc-afni', action='store_true', default=True, help='Use ANFI 3dvolreg for head motion correction (HMC)') g_input.add_argument( '--hmc-fsl', action='store_true', default=False, help='Use FSL MCFLIRT for head motion correction (HMC)') g_input.add_argument( '-f', '--float32', action='store_true', default=DEFAULTS['float32'], help= "Cast the input data to float32 if it's represented in higher precision " "(saves space and improves perfomance)") g_input.add_argument('--fft-spikes-detector', action='store_true', default=False, help='Turn on FFT based spike detector (slow).') g_outputs = parser.add_argument_group('mriqc specific outputs') g_outputs.add_argument('-w', '--work-dir', action='store', default=op.join(os.getcwd(), 'work')) g_outputs.add_argument('--report-dir', action='store') g_outputs.add_argument('--verbose-reports', default=False, action='store_true') # ANTs options g_ants = parser.add_argument_group( 'specific settings for ANTs registrations') g_ants.add_argument( '--ants-nthreads', action='store', type=int, default=DEFAULTS['ants_nthreads'], help='number of threads that will be set in ANTs processes') g_ants.add_argument('--ants-settings', action='store', help='path to JSON file with settings for ANTS') # AFNI head motion correction settings g_afni = parser.add_argument_group( 'specific settings for AFNI head motion correction') g_afni.add_argument( '--deoblique', action='store_true', default=False, help='Deoblique the functional scans during head motion ' 'correction preprocessing') g_afni.add_argument( '--despike', action='store_true', default=False, help='Despike the functional scans during head motion correction ' 'preprocessing') g_afni.add_argument( '--start-idx', action='store', type=int, help='Initial volume in functional timeseries that should be ' 'considered for preprocessing') g_afni.add_argument( '--stop-idx', action='store', type=int, help='Final volume in functional timeseries that should be ' 'considered for preprocessing') g_afni.add_argument('--correct-slice-timing', action='store_true', default=False, help='Perform slice timing correction') opts = parser.parse_args() # Build settings dict bids_dir = op.abspath(opts.bids_dir) # Number of processes n_procs = 0 if opts.nthreads is not None: MRIQC_LOG.warn('Option --nthreads has been deprecated in mriqc 0.8.8. ' 'Please use --n_procs instead.') n_procs = opts.nthreads if opts.n_procs is not None: n_procs = opts.n_procs # Check physical memory total_memory = opts.mem_gb if total_memory < 0: try: from psutil import virtual_memory total_memory = virtual_memory().total // (1024**3) + 1 except ImportError: MRIQC_LOG.warn( 'Total physical memory could not be estimated, using %d' 'GB as default', DEFAULT_MEM_GB) total_memory = DEFAULT_MEM_GB if total_memory > 0: av_procs = total_memory // 4 if av_procs < 1: MRIQC_LOG.warn( 'Total physical memory is less than 4GB, memory allocation' ' problems are likely to occur.') n_procs = 1 elif n_procs > av_procs: n_procs = av_procs settings = { 'bids_dir': bids_dir, 'write_graph': opts.write_graph, 'testing': opts.testing, 'hmc_afni': opts.hmc_afni, 'hmc_fsl': opts.hmc_fsl, 'fft_spikes_detector': opts.fft_spikes_detector, 'n_procs': n_procs, 'ants_nthreads': opts.ants_nthreads, 'output_dir': op.abspath(opts.output_dir), 'work_dir': op.abspath(opts.work_dir), 'verbose_reports': opts.verbose_reports or opts.testing, 'float32': opts.float32 } if opts.hmc_afni: settings['deoblique'] = opts.deoblique settings['despike'] = opts.despike settings['correct_slice_timing'] = opts.correct_slice_timing if opts.start_idx: settings['start_idx'] = opts.start_idx if opts.stop_idx: settings['stop_idx'] = opts.stop_idx if opts.ants_settings: settings['ants_settings'] = opts.ants_settings log_dir = op.join(settings['output_dir'], 'logs') analysis_levels = opts.analysis_level if opts.participant_label is None: analysis_levels.append('group') analysis_levels = list(set(analysis_levels)) if len(analysis_levels) > 2: raise RuntimeError('Error parsing analysis levels, got "%s"' % ', '.join(analysis_levels)) settings['report_dir'] = opts.report_dir if not settings['report_dir']: settings['report_dir'] = op.join(settings['output_dir'], 'reports') check_folder(settings['output_dir']) if 'participant' in analysis_levels: check_folder(settings['work_dir']) check_folder(log_dir) check_folder(settings['report_dir']) # Set nipype config ncfg.update_config({ 'logging': { 'log_directory': log_dir, 'log_to_file': True }, 'execution': { 'crashdump_dir': log_dir } }) plugin_settings = {'plugin': 'Linear'} if opts.use_plugin is not None: from yaml import load as loadyml with open(opts.use_plugin) as pfile: plugin_settings = loadyml(pfile) else: # Setup multiprocessing if settings['n_procs'] == 0: settings['n_procs'] = 1 max_parallel_ants = cpu_count() // settings['ants_nthreads'] if max_parallel_ants > 1: settings['n_procs'] = max_parallel_ants if settings['n_procs'] > 1: plugin_settings['plugin'] = 'MultiProc' plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']} MRIQC_LOG.info( 'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s', __version__, ', '.join(analysis_levels), opts.participant_label, settings) # Process data types modalities = opts.modalities dataset = collect_bids_data(settings['bids_dir'], participant_label=opts.participant_label) # Set up participant level if 'participant' in analysis_levels: workflow = Workflow(name='workflow_enumerator') workflow.base_dir = settings['work_dir'] wf_list = [] for mod in modalities: if not dataset[mod]: MRIQC_LOG.warn('No %s scans were found in %s', mod, settings['bids_dir']) continue wf_list.append(build_workflow(dataset[mod], mod, settings=settings)) if wf_list: workflow.add_nodes(wf_list) if not opts.dry_run: workflow.run(**plugin_settings) else: raise RuntimeError( 'Error reading BIDS directory (%s), or the dataset is not ' 'BIDS-compliant.' % settings['bids_dir']) # Set up group level if 'group' in analysis_levels: from mriqc.reports import group_html from mriqc.utils.misc import generate_csv, generate_pred reports_dir = check_folder(op.join(settings['output_dir'], 'reports')) derivatives_dir = op.join(settings['output_dir'], 'derivatives') n_group_reports = 0 for mod in modalities: dataframe, out_csv = generate_csv(derivatives_dir, settings['output_dir'], mod) # If there are no iqm.json files, nothing to do. if dataframe is None: MRIQC_LOG.warn( 'No IQM-JSON files were found for the %s data type in %s. The group-level ' 'report was not generated.', mod, derivatives_dir) continue MRIQC_LOG.info('Summary CSV table for the %s data generated (%s)', mod, out_csv) out_pred = generate_pred(derivatives_dir, settings['output_dir'], mod) if out_pred is not None: MRIQC_LOG.info( 'Predicted QA CSV table for the %s data generated (%s)', mod, out_pred) out_html = op.join(reports_dir, mod + '_group.html') group_html(out_csv, mod, csv_failed=op.join(settings['output_dir'], 'failed_' + mod + '.csv'), out_file=out_html) MRIQC_LOG.info('Group-%s report generated (%s)', mod, out_html) n_group_reports += 1 if n_group_reports == 0: raise Exception( "No data found. No group level reports were generated.")
def create_workflow(wf_base_dir, input_anat, oasis_path): ''' Method to create the nipype workflow that is executed for preprocessing the data Parameters ---------- wf_base_dir : string filepath to the base directory to run the workflow input_anat : string filepath to the input file to run antsCorticalThickness.sh on oasis_path : string filepath to the oasis Returns ------- wf : nipype.pipeline.engine.Workflow instance the workflow to be ran for preprocessing ''' # Import packages from act_interface import antsCorticalThickness import nipype.interfaces.io as nio import nipype.pipeline.engine as pe import nipype.interfaces.utility as util from nipype.interfaces.utility import Function from nipype import logging as np_logging from nipype import config import os # Init variables oasis_trt_20 = os.path.join(oasis_path, 'OASIS-TRT-20_jointfusion_DKT31_CMA_labels_in_OASIS-30.nii') # Setup nipype workflow if not os.path.exists(wf_base_dir): os.makedirs(wf_base_dir) wf = pe.Workflow(name='thickness_workflow') wf.base_dir = wf_base_dir # Init log directory log_dir = wf_base_dir # Define antsCorticalThickness node thickness = pe.Node(antsCorticalThickness(), name='thickness') # Set antsCorticalThickness inputs thickness.inputs.dimension = 3 thickness.inputs.segmentation_iterations = 1 thickness.inputs.segmentation_weight = 0.25 thickness.inputs.input_skull = input_anat #-a thickness.inputs.template = oasis_path + 'T_template0.nii.gz' #-e thickness.inputs.brain_prob_mask = oasis_path + \ 'T_template0_BrainCerebellumProbabilityMask.nii.gz' #-m thickness.inputs.brain_seg_priors = oasis_path + \ 'Priors2/priors%d.nii.gz' #-p thickness.inputs.intensity_template = oasis_path + \ 'T_template0_BrainCerebellum.nii.gz' #-t thickness.inputs.extraction_registration_mask = oasis_path + \ 'T_template0_BrainCerebellumExtractionMask.nii.gz' #-f thickness.inputs.out_prefix = 'OUTPUT_' #-o thickness.inputs.keep_intermediate_files = 0 #-k # Node to run ANTs 3dROIStats ROIstats = pe.Node(util.Function(input_names=['mask','thickness_normd'], output_names=['roi_stats_file'], function=roi_func), name='ROIstats') wf.connect(thickness, 'cortical_thickness_normalized', ROIstats, 'thickness_normd') ROIstats.inputs.mask = oasis_trt_20 # Create datasink node datasink = pe.Node(nio.DataSink(), name='sinker') datasink.inputs.base_directory = wf_base_dir # Connect thickness outputs to datasink wf.connect(thickness, 'brain_extraction_mask', datasink, 'output.@brain_extr_mask') wf.connect(thickness, 'brain_segmentation', datasink, 'output.@brain_seg') wf.connect(thickness, 'brain_segmentation_N4', datasink, 'output.@brain_seg_N4') wf.connect(thickness, 'brain_segmentation_posteriors_1', datasink, 'output.@brain_seg_post_1') wf.connect(thickness, 'brain_segmentation_posteriors_2', datasink, 'output.@brain_seg_post_2') wf.connect(thickness, 'brain_segmentation_posteriors_3', datasink, 'output.@brain_seg_post_3') wf.connect(thickness, 'brain_segmentation_posteriors_4', datasink, 'output.@brain_seg_post_4') wf.connect(thickness, 'brain_segmentation_posteriors_5', datasink, 'output.@brain_seg_post_5') wf.connect(thickness, 'brain_segmentation_posteriors_6', datasink, 'output.@brain_seg_post_6') wf.connect(thickness, 'cortical_thickness', datasink, 'output.@cortical_thickness') wf.connect(thickness, 'cortical_thickness_normalized', datasink,'output.@cortical_thickness_normalized') # Connect ROI stats output text file to datasink wf.connect(ROIstats, 'roi_stats_file', datasink, 'output.@ROIstats') # Setup crashfile directory and logging wf.config['execution'] = {'hash_method': 'timestamp', 'crashdump_dir': '/home/ubuntu/crashes'} config.update_config({'logging': {'log_directory': log_dir, 'log_to_file': True}}) np_logging.update_logging(config) # Return the workflow return wf
#-------------# # Load packages import nipype.interfaces.io as nio # Data i/o import nipype.interfaces.fsl as fsl # fsl import nipype.interfaces.utility as util # utility import nipype.pipeline.engine as pe # pypeline engine import os # system functions import procfunc as func # config from nipype import config cfg = dict(logging=dict(workflow_level = 'DEBUG'), execution={'stop_on_first_crash': False, 'hash_method': 'timestamp'}) # content config.update_config(cfg) # Info source info = dict(dwi=[['dwi_dirs']], bvecs=[['dwi_dirs']], bvals=[['dwi_dirs']]) infosource = pe.Node(interface=util.IdentityInterface(fields=['dwi_dirs']),name='infosource') infosource.iterables = ('dwi_dirs', dwiDirectories) # Data grabber DWI datasource_dwi = pe.Node(interface=nio.DataGrabber(infields=['dwi_dirs'], outfields=info.keys()), name = 'datasource') datasource_dwi.inputs.base_directory = dataDirectory datasource_dwi.inputs.template = '*' datasource_dwi.inputs.field_template = dict(dwi='%s/*.nii*',
def calc_centrality_metrics(cfg): import os from nipype import config from nipype.pipeline.engine import Node, Workflow import nipype.interfaces.utility as util import nipype.interfaces.io as nio import nipype.interfaces.fsl as fsl import nipype.interfaces.freesurfer as freesurfer import CPAC.network_centrality.resting_state_centrality as cpac_centrality import CPAC.network_centrality.z_score as cpac_centrality_z_score # INPUT PARAMETERS dicom_dir = cfg['dicom_dir'] preprocessed_data_dir = cfg['preprocessed_data_dir'] working_dir = cfg['working_dir'] freesurfer_dir = cfg['freesurfer_dir'] template_dir = cfg['template_dir'] script_dir = cfg['script_dir'] ds_dir = cfg['ds_dir'] subjects_list = cfg['subjects_list'] TR_list = cfg['TR_list'] use_n_procs = cfg['use_n_procs'] plugin_name = cfg['plugin_name'] ##################################### # GENERAL SETTINGS ##################################### fsl.FSLCommand.set_default_output_type('NIFTI_GZ') freesurfer.FSCommand.set_default_subjects_dir(freesurfer_dir) wf = Workflow(name='LeiCA_metrics') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': False, 'remove_unnecessary_outputs': True, 'job_finished_timeout': 120}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash') ds = Node(nio.DataSink(), name='ds') ds.inputs.substitutions = [('_TR_id_', 'TR_')] ds.inputs.regexp_substitutions = [('_subject_id_[A0-9]*/', ''), ( '_z_score[0-9]*/', '')] # , #('dc/_TR_id_[0-9]*/', ''), ('evc/_TR_id_[0-9]*/','')] ##################################### # SET ITERATORS ##################################### # GET SCAN TR_ID ITERATOR scan_infosource = Node(util.IdentityInterface(fields=['TR_id']), name='scan_infosource') scan_infosource.iterables = ('TR_id', TR_list) subjects_infosource = Node(util.IdentityInterface(fields=['subject_id']), name='subjects_infosource') subjects_infosource.iterables = ('subject_id', subjects_list) def add_subject_id_to_ds_dir_fct(subject_id, ds_path): import os out_path = os.path.join(ds_path, subject_id) return out_path add_subject_id_to_ds_dir = Node(util.Function(input_names=['subject_id', 'ds_path'], output_names=['out_path'], function=add_subject_id_to_ds_dir_fct), name='add_subject_id_to_ds_dir') wf.connect(subjects_infosource, 'subject_id', add_subject_id_to_ds_dir, 'subject_id') add_subject_id_to_ds_dir.inputs.ds_path = ds_dir wf.connect(add_subject_id_to_ds_dir, 'out_path', ds, 'base_directory') # get atlas data templates_atlases = {'GM_mask_MNI_2mm': 'SPM_GM/SPM_GM_mask_2mm.nii.gz', 'GM_mask_MNI_3mm': 'SPM_GM/SPM_GM_mask_3mm.nii.gz', 'FSL_MNI_3mm_template': 'MNI152_T1_3mm_brain.nii.gz', 'vmhc_symm_brain': 'cpac_image_resources/symmetric/MNI152_T1_2mm_brain_symmetric.nii.gz', 'vmhc_symm_brain_3mm': 'cpac_image_resources/symmetric/MNI152_T1_3mm_brain_symmetric.nii.gz', 'vmhc_symm_skull': 'cpac_image_resources/symmetric/MNI152_T1_2mm_symmetric.nii.gz', 'vmhc_symm_brain_mask_dil': 'cpac_image_resources/symmetric/MNI152_T1_2mm_brain_mask_symmetric_dil.nii.gz', 'vmhc_config_file_2mm': 'cpac_image_resources/symmetric/T1_2_MNI152_2mm_symmetric.cnf' } selectfiles_anat_templates = Node(nio.SelectFiles(templates_atlases, base_directory=template_dir), name="selectfiles_anat_templates") # GET SUBJECT SPECIFIC FUNCTIONAL AND STRUCTURAL DATA selectfiles_templates = { 'epi_2_MNI_warp': '{subject_id}/rsfMRI_preprocessing/registration/epi_2_MNI_warp/TR_{TR_id}/*.nii.gz', 'epi_mask': '{subject_id}/rsfMRI_preprocessing/masks/brain_mask_epiSpace/TR_{TR_id}/*.nii.gz', 'preproc_epi_full_spectrum': '{subject_id}/rsfMRI_preprocessing/epis/01_denoised/TR_{TR_id}/*.nii.gz', 'preproc_epi_bp': '{subject_id}/rsfMRI_preprocessing/epis/02_denoised_BP/TR_{TR_id}/*.nii.gz', 'preproc_epi_bp_tNorm': '{subject_id}/rsfMRI_preprocessing/epis/03_denoised_BP_tNorm/TR_{TR_id}/*.nii.gz', 'epi_2_struct_mat': '{subject_id}/rsfMRI_preprocessing/registration/epi_2_struct_mat/TR_{TR_id}/*.mat', 't1w': '{subject_id}/raw_niftis/sMRI/t1w_reoriented.nii.gz', 't1w_brain': '{subject_id}/rsfMRI_preprocessing/struct_prep/t1w_brain/t1w_reoriented_maths.nii.gz', 'epi_bp_tNorm_MNIspace_3mm': '{subject_id}/rsfMRI_preprocessing/epis_MNI_3mm/03_denoised_BP_tNorm/TR_645/residual_filt_norm_warp.nii.gz' } selectfiles = Node(nio.SelectFiles(selectfiles_templates, base_directory=preprocessed_data_dir), name="selectfiles") wf.connect(scan_infosource, 'TR_id', selectfiles, 'TR_id') wf.connect(subjects_infosource, 'subject_id', selectfiles, 'subject_id') # selectfiles.inputs.subject_id = subject_id # CREATE TRANSFORMATIONS # creat MNI 2 epi warp MNI_2_epi_warp = Node(fsl.InvWarp(), name='MNI_2_epi_warp') MNI_2_epi_warp.inputs.reference = fsl.Info.standard_image('MNI152_T1_2mm.nii.gz') wf.connect(selectfiles, 'epi_mask', MNI_2_epi_warp, 'reference') wf.connect(selectfiles, 'epi_2_MNI_warp', MNI_2_epi_warp, 'warp') ##################### # CALCULATE METRICS ##################### # DEGREE # fixme # a_mem = 5 # fixme a_mem = 20 dc = cpac_centrality.create_resting_state_graphs(allocated_memory=a_mem, wf_name='dc') # allocated_memory = a_mem, wf_name = 'dc') # dc.plugin_args = {'submit_specs': 'request_memory = 6000'} # fixme dc.plugin_args = {'submit_specs': 'request_memory = 20000'} dc.inputs.inputspec.method_option = 0 # 0 for degree centrality, 1 for eigenvector centrality, 2 for lFCD dc.inputs.inputspec.threshold_option = 0 # 0 for probability p_value, 1 for sparsity threshold, any other for threshold value dc.inputs.inputspec.threshold = 0.0001 dc.inputs.inputspec.weight_options = [True, True] # list of two booleans for binarize and weighted options respectively wf.connect(selectfiles, 'epi_bp_tNorm_MNIspace_3mm', dc, 'inputspec.subject') wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', dc, 'inputspec.template') wf.connect(dc, 'outputspec.centrality_outputs', ds, 'metrics.centrality.dc.@centrality_outputs') wf.connect(dc, 'outputspec.correlation_matrix', ds, 'metrics.centrality.dc.@correlation_matrix') wf.connect(dc, 'outputspec.graph_outputs', ds, 'metrics.centrality.dc.@graph_outputs') # DC Z-SCORE dc_Z = cpac_centrality_z_score.get_cent_zscore(wf_name='dc_Z') wf.connect(dc, 'outputspec.centrality_outputs', dc_Z, 'inputspec.input_file') wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', dc_Z, 'inputspec.mask_file') wf.connect(dc_Z, 'outputspec.z_score_img', ds, 'metrics.centrality.dc_z.@output') a_mem = 20 evc = cpac_centrality.create_resting_state_graphs(allocated_memory=a_mem, wf_name='evc') evc.plugin_args = {'submit_specs': 'request_memory = 20000'} evc.inputs.inputspec.method_option = 1 # 0 for degree centrality, 1 for eigenvector centrality, 2 for lFCD evc.inputs.inputspec.threshold_option = 0 # 0 for probability p_value, 1 for sparsity threshold, any other for threshold value evc.inputs.inputspec.threshold = 0.0001 evc.inputs.inputspec.weight_options = [True, True] # list of two booleans for binarize and weighted options respectively wf.connect(selectfiles, 'epi_bp_tNorm_MNIspace_3mm', evc, 'inputspec.subject') wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', evc, 'inputspec.template') wf.connect(evc, 'outputspec.centrality_outputs', ds, 'metrics.centrality.evc.@centrality_outputs') wf.connect(evc, 'outputspec.correlation_matrix', ds, 'metrics.centrality.evc.@correlation_matrix') wf.connect(evc, 'outputspec.graph_outputs', ds, 'metrics.centrality.evc.@graph_outputs') # EVC Z-SCORE evc_Z = cpac_centrality_z_score.get_cent_zscore(wf_name='evc_Z') wf.connect(evc, 'outputspec.centrality_outputs', evc_Z, 'inputspec.input_file') wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', evc_Z, 'inputspec.mask_file') wf.connect(evc_Z, 'outputspec.z_score_img', ds, 'metrics.centrality.evc_z.@output') wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def build_workflow(opts, retval): """ Create the Nipype Workflow that supports the whole execution graph, given the inputs. All the checks and the construction of the workflow are done inside this function that has pickleable inputs and output dictionary (``retval``) to allow isolation using a ``multiprocessing.Process`` that allows smriprep to enforce a hard-limited memory-scope. """ from shutil import copyfile from os import cpu_count import uuid from time import strftime from subprocess import check_call, CalledProcessError, TimeoutExpired from pkg_resources import resource_filename as pkgrf import json from bids import BIDSLayout from nipype import logging, config as ncfg from niworkflows.utils.bids import collect_participants from ..__about__ import __version__ from ..workflows.base import init_smriprep_wf logger = logging.getLogger('nipype.workflow') INIT_MSG = """ Running sMRIPrep version {version}: * BIDS dataset path: {bids_dir}. * Participant list: {subject_list}. * Run identifier: {uuid}. {spaces} """.format # Set up some instrumental utilities run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4()) # First check that bids_dir looks like a BIDS folder bids_dir = opts.bids_dir.resolve() layout = BIDSLayout(str(bids_dir), validate=False) subject_list = collect_participants( layout, participant_label=opts.participant_label) bids_filters = json.loads( opts.bids_filter_file.read_text()) if opts.bids_filter_file else None # Load base plugin_settings from file if --use-plugin if opts.use_plugin is not None: from yaml import load as loadyml with open(opts.use_plugin) as f: plugin_settings = loadyml(f) plugin_settings.setdefault('plugin_args', {}) else: # Defaults plugin_settings = { 'plugin': 'MultiProc', 'plugin_args': { 'raise_insufficient': False, 'maxtasksperchild': 1, } } # Resource management options # Note that we're making strong assumptions about valid plugin args # This may need to be revisited if people try to use batch plugins nprocs = plugin_settings['plugin_args'].get('n_procs') # Permit overriding plugin config with specific CLI options if nprocs is None or opts.nprocs is not None: nprocs = opts.nprocs if nprocs is None or nprocs < 1: nprocs = cpu_count() plugin_settings['plugin_args']['n_procs'] = nprocs if opts.mem_gb: plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb omp_nthreads = opts.omp_nthreads if omp_nthreads == 0: omp_nthreads = min(nprocs - 1 if nprocs > 1 else cpu_count(), 8) if 1 < nprocs < omp_nthreads: logger.warning( 'Per-process threads (--omp-nthreads=%d) exceed total ' 'available CPUs (--nprocs/--ncpus=%d)', omp_nthreads, nprocs) # Set up directories output_dir = opts.output_dir.resolve() log_dir = output_dir / 'smriprep' / 'logs' work_dir = opts.work_dir.resolve() # Check and create output and working directories log_dir.mkdir(parents=True, exist_ok=True) work_dir.mkdir(parents=True, exist_ok=True) # Nipype config (logs and execution) ncfg.update_config({ 'logging': { 'log_directory': str(log_dir), 'log_to_file': True }, 'execution': { 'crashdump_dir': str(log_dir), 'crashfile_format': 'txt', 'get_linked_libs': False, 'stop_on_first_crash': opts.stop_on_first_crash, }, 'monitoring': { 'enabled': opts.resource_monitor, 'sample_frequency': '0.5', 'summary_append': True, } }) if opts.resource_monitor: ncfg.enable_resource_monitor() retval['return_code'] = 0 retval['plugin_settings'] = plugin_settings retval['bids_dir'] = str(bids_dir) retval['output_dir'] = str(output_dir) retval['work_dir'] = str(work_dir) retval['subject_list'] = subject_list retval['run_uuid'] = run_uuid retval['workflow'] = None # Called with reports only if opts.reports_only: from niworkflows.reports import generate_reports logger.log(25, 'Running --reports-only on participants %s', ', '.join(subject_list)) if opts.run_uuid is not None: run_uuid = opts.run_uuid retval['return_code'] = generate_reports(subject_list, str(output_dir), run_uuid, packagename="smriprep") return retval logger.log( 25, INIT_MSG(version=__version__, bids_dir=bids_dir, subject_list=subject_list, uuid=run_uuid, spaces=opts.output_spaces)) # Build main workflow retval['workflow'] = init_smriprep_wf( debug=opts.sloppy, fast_track=opts.fast_track, freesurfer=opts.run_reconall, fs_subjects_dir=opts.fs_subjects_dir, hires=opts.hires, layout=layout, longitudinal=opts.longitudinal, low_mem=opts.low_mem, omp_nthreads=omp_nthreads, output_dir=str(output_dir), run_uuid=run_uuid, skull_strip_fixed_seed=opts.skull_strip_fixed_seed, skull_strip_mode=opts.skull_strip_mode, skull_strip_template=opts.skull_strip_template[0], spaces=opts.output_spaces, subject_list=subject_list, work_dir=str(work_dir), bids_filters=bids_filters, ) retval['return_code'] = 0 boilerplate = retval['workflow'].visit_desc() (log_dir / 'CITATION.md').write_text(boilerplate) logger.log( 25, 'Works derived from this sMRIPrep execution should ' 'include the following boilerplate:\n\n%s', boilerplate) # Generate HTML file resolving citations cmd = [ 'pandoc', '-s', '--bibliography', pkgrf('smriprep', 'data/boilerplate.bib'), '--filter', 'pandoc-citeproc', '--metadata', 'pagetitle="sMRIPrep citation boilerplate"', str(log_dir / 'CITATION.md'), '-o', str(log_dir / 'CITATION.html') ] try: check_call(cmd, timeout=10) except (FileNotFoundError, CalledProcessError, TimeoutExpired): logger.warning('Could not generate CITATION.html file:\n%s', ' '.join(cmd)) # Generate LaTex file resolving citations cmd = [ 'pandoc', '-s', '--bibliography', pkgrf('smriprep', 'data/boilerplate.bib'), '--natbib', str(log_dir / 'CITATION.md'), '-o', str(log_dir / 'CITATION.tex') ] try: check_call(cmd, timeout=10) except (FileNotFoundError, CalledProcessError, TimeoutExpired): logger.warning('Could not generate CITATION.tex file:\n%s', ' '.join(cmd)) else: copyfile(pkgrf('smriprep', 'data/boilerplate.bib'), str(log_dir / 'CITATION.bib')) return retval
def calc_local_metrics(cfg): import os from nipype import config from nipype.pipeline.engine import Node, Workflow, MapNode import nipype.interfaces.utility as util import nipype.interfaces.io as nio import nipype.interfaces.fsl as fsl import nipype.interfaces.freesurfer as freesurfer import CPAC.alff.alff as cpac_alff import CPAC.reho.reho as cpac_reho import CPAC.utils.utils as cpac_utils import CPAC.vmhc.vmhc as cpac_vmhc import CPAC.registration.registration as cpac_registration import CPAC.network_centrality.z_score as cpac_centrality_z_score import utils as calc_metrics_utils # INPUT PARAMETERS dicom_dir = cfg['dicom_dir'] preprocessed_data_dir = cfg['preprocessed_data_dir'] working_dir = cfg['working_dir'] freesurfer_dir = cfg['freesurfer_dir'] template_dir = cfg['template_dir'] script_dir = cfg['script_dir'] ds_dir = cfg['ds_dir'] subject_id = cfg['subject_id'] TR_list = cfg['TR_list'] vols_to_drop = cfg['vols_to_drop'] rois_list = cfg['rois_list'] lp_cutoff_freq = cfg['lp_cutoff_freq'] hp_cutoff_freq = cfg['hp_cutoff_freq'] use_fs_brainmask = cfg['use_fs_brainmask'] use_n_procs = cfg['use_n_procs'] plugin_name = cfg['plugin_name'] ##################################### # GENERAL SETTINGS ##################################### fsl.FSLCommand.set_default_output_type('NIFTI_GZ') freesurfer.FSCommand.set_default_subjects_dir(freesurfer_dir) wf = Workflow(name='LeiCA_metrics') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True, 'remove_unnecessary_outputs': True, 'job_finished_timeout': 120}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash') ds = Node(nio.DataSink(base_directory=ds_dir), name='ds') ds.inputs.substitutions = [('_TR_id_', 'TR_')] ds.inputs.regexp_substitutions = [('_variabilty_MNIspace_3mm[0-9]*/', ''), ('_z_score[0-9]*/', '')] ##################################### # SET ITERATORS ##################################### # GET SCAN TR_ID ITERATOR scan_infosource = Node(util.IdentityInterface(fields=['TR_id']), name='scan_infosource') scan_infosource.iterables = ('TR_id', TR_list) # get atlas data templates_atlases = { # 'GM_mask_MNI_2mm': 'SPM_GM/SPM_GM_mask_2mm.nii.gz', # 'GM_mask_MNI_3mm': 'SPM_GM/SPM_GM_mask_3mm.nii.gz', 'FSL_MNI_3mm_template': 'MNI152_T1_3mm_brain.nii.gz', 'vmhc_symm_brain': 'cpac_image_resources/symmetric/MNI152_T1_2mm_brain_symmetric.nii.gz', 'vmhc_symm_brain_3mm': 'cpac_image_resources/symmetric/MNI152_T1_3mm_brain_symmetric.nii.gz', 'vmhc_symm_skull': 'cpac_image_resources/symmetric/MNI152_T1_2mm_symmetric.nii.gz', 'vmhc_symm_brain_mask_dil': 'cpac_image_resources/symmetric/MNI152_T1_2mm_brain_mask_symmetric_dil.nii.gz', 'vmhc_config_file_2mm': 'cpac_image_resources/symmetric/T1_2_MNI152_2mm_symmetric.cnf' } selectfiles_anat_templates = Node(nio.SelectFiles(templates_atlases, base_directory=template_dir), name="selectfiles_anat_templates") # GET SUBJECT SPECIFIC FUNCTIONAL AND STRUCTURAL DATA selectfiles_templates = { 'epi_2_MNI_warp': '{subject_id}/rsfMRI_preprocessing/registration/epi_2_MNI_warp/TR_{TR_id}/*.nii.gz', 'epi_mask': '{subject_id}/rsfMRI_preprocessing/masks/brain_mask_epiSpace/TR_{TR_id}/*.nii.gz', 'preproc_epi_full_spectrum': '{subject_id}/rsfMRI_preprocessing/epis/01_denoised/TR_{TR_id}/*.nii.gz', 'preproc_epi_bp': '{subject_id}/rsfMRI_preprocessing/epis/02_denoised_BP/TR_{TR_id}/*.nii.gz', 'preproc_epi_bp_tNorm': '{subject_id}/rsfMRI_preprocessing/epis/03_denoised_BP_tNorm/TR_{TR_id}/*.nii.gz', 'epi_2_struct_mat': '{subject_id}/rsfMRI_preprocessing/registration/epi_2_struct_mat/TR_{TR_id}/*.mat', 't1w': '{subject_id}/raw_niftis/sMRI/t1w_reoriented.nii.gz', 't1w_brain': '{subject_id}/rsfMRI_preprocessing/struct_prep/t1w_brain/t1w_reoriented_maths.nii.gz', } selectfiles = Node(nio.SelectFiles(selectfiles_templates, base_directory=preprocessed_data_dir), name="selectfiles") wf.connect(scan_infosource, 'TR_id', selectfiles, 'TR_id') selectfiles.inputs.subject_id = subject_id # CREATE TRANSFORMATIONS # creat MNI 2 epi warp MNI_2_epi_warp = Node(fsl.InvWarp(), name='MNI_2_epi_warp') MNI_2_epi_warp.inputs.reference = fsl.Info.standard_image('MNI152_T1_2mm.nii.gz') wf.connect(selectfiles, 'epi_mask', MNI_2_epi_warp, 'reference') wf.connect(selectfiles, 'epi_2_MNI_warp', MNI_2_epi_warp, 'warp') # # CREATE GM MASK IN EPI SPACE # GM_mask_epiSpace = Node(fsl.ApplyWarp(), name='GM_mask_epiSpace') # GM_mask_epiSpace.inputs.out_file = 'GM_mask_epiSpace.nii.gz' # # wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_2mm', GM_mask_epiSpace, 'in_file') # wf.connect(selectfiles, 'epi_mask', GM_mask_epiSpace, 'ref_file') # wf.connect(MNI_2_epi_warp, 'inverse_warp', GM_mask_epiSpace, 'field_file') # wf.connect(GM_mask_epiSpace, 'out_file', ds, 'GM_mask_epiSpace') # fixme # # CREATE TS IN MNI SPACE # # is it ok to apply the 2mm warpfield to the 3mm template? # # seems ok: https://www.jiscmail.ac.uk/cgi-bin/webadmin?A2=ind0904&L=FSL&P=R14011&1=FSL&9=A&J=on&d=No+Match%3BMatch%3BMatches&z=4 # epi_bp_MNIspace_3mm = Node(fsl.ApplyWarp(), name='epi_bp_MNIspace_3mm') # epi_bp_MNIspace_3mm.inputs.interp = 'spline' # epi_bp_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'} # wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', epi_bp_MNIspace_3mm, 'ref_file') # wf.connect(selectfiles, 'preproc_epi_bp', epi_bp_MNIspace_3mm, 'in_file') # wf.connect(selectfiles, 'epi_2_MNI_warp', epi_bp_MNIspace_3mm, 'field_file') # CREATE EPI MASK IN MNI SPACE epi_mask_MNIspace_3mm = Node(fsl.ApplyWarp(), name='epi_mask_MNIspace_3mm') epi_mask_MNIspace_3mm.inputs.interp = 'nn' epi_mask_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'} wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', epi_mask_MNIspace_3mm, 'ref_file') wf.connect(selectfiles, 'epi_mask', epi_mask_MNIspace_3mm, 'in_file') wf.connect(selectfiles, 'epi_2_MNI_warp', epi_mask_MNIspace_3mm, 'field_file') wf.connect(epi_mask_MNIspace_3mm, 'out_file', ds, 'epi_mask_MNIspace_3mm') ##################### # CALCULATE METRICS ##################### # f/ALFF alff = cpac_alff.create_alff('alff') alff.inputs.hp_input.hp = 0.01 alff.inputs.lp_input.lp = 0.1 wf.connect(selectfiles, 'preproc_epi_full_spectrum', alff, 'inputspec.rest_res') # wf.connect(GM_mask_epiSpace, 'out_file', alff, 'inputspec.rest_mask') wf.connect(selectfiles, 'epi_mask', alff, 'inputspec.rest_mask') wf.connect(alff, 'outputspec.alff_img', ds, 'alff.alff') wf.connect(alff, 'outputspec.falff_img', ds, 'alff.falff') # f/ALFF 2 MNI # fixme spline or default? alff_MNIspace_3mm = Node(fsl.ApplyWarp(), name='alff_MNIspace_3mm') alff_MNIspace_3mm.inputs.interp = 'spline' alff_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'} wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', alff_MNIspace_3mm, 'ref_file') wf.connect(alff, 'outputspec.alff_img', alff_MNIspace_3mm, 'in_file') wf.connect(selectfiles, 'epi_2_MNI_warp', alff_MNIspace_3mm, 'field_file') wf.connect(alff_MNIspace_3mm, 'out_file', ds, 'alff.alff_MNI_3mm') falff_MNIspace_3mm = Node(fsl.ApplyWarp(), name='falff_MNIspace_3mm') falff_MNIspace_3mm.inputs.interp = 'spline' falff_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'} wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', falff_MNIspace_3mm, 'ref_file') wf.connect(alff, 'outputspec.falff_img', falff_MNIspace_3mm, 'in_file') wf.connect(selectfiles, 'epi_2_MNI_warp', falff_MNIspace_3mm, 'field_file') wf.connect(falff_MNIspace_3mm, 'out_file', ds, 'alff.falff_MNI_3mm') # f/ALFF_MNI Z-SCORE alff_MNIspace_3mm_Z = cpac_utils.get_zscore(input_name='alff_MNIspace_3mm', wf_name='alff_MNIspace_3mm_Z') wf.connect(alff_MNIspace_3mm, 'out_file', alff_MNIspace_3mm_Z, 'inputspec.input_file') # wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', alff_MNIspace_3mm_Z, 'inputspec.mask_file') wf.connect(epi_mask_MNIspace_3mm, 'out_file', alff_MNIspace_3mm_Z, 'inputspec.mask_file') wf.connect(alff_MNIspace_3mm_Z, 'outputspec.z_score_img', ds, 'alff.alff_MNI_3mm_Z') falff_MNIspace_3mm_Z = cpac_utils.get_zscore(input_name='falff_MNIspace_3mm', wf_name='falff_MNIspace_3mm_Z') wf.connect(falff_MNIspace_3mm, 'out_file', falff_MNIspace_3mm_Z, 'inputspec.input_file') # wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', falff_MNIspace_3mm_Z, 'inputspec.mask_file') wf.connect(epi_mask_MNIspace_3mm, 'out_file', falff_MNIspace_3mm_Z, 'inputspec.mask_file') wf.connect(falff_MNIspace_3mm_Z, 'outputspec.z_score_img', ds, 'alff.falff_MNI_3mm_Z') # f/ALFF_MNI STANDARDIZE BY MEAN alff_MNIspace_3mm_standardized_mean = calc_metrics_utils.standardize_divide_by_mean( wf_name='alff_MNIspace_3mm_standardized_mean') wf.connect(alff_MNIspace_3mm, 'out_file', alff_MNIspace_3mm_standardized_mean, 'inputnode.in_file') wf.connect(epi_mask_MNIspace_3mm, 'out_file', alff_MNIspace_3mm_standardized_mean, 'inputnode.mask_file') wf.connect(alff_MNIspace_3mm_standardized_mean, 'outputnode.out_file', ds, 'alff.alff_MNI_3mm_standardized_mean') falff_MNIspace_3mm_standardized_mean = calc_metrics_utils.standardize_divide_by_mean( wf_name='falff_MNIspace_3mm_standardized_mean') wf.connect(falff_MNIspace_3mm, 'out_file', falff_MNIspace_3mm_standardized_mean, 'inputnode.in_file') wf.connect(epi_mask_MNIspace_3mm, 'out_file', falff_MNIspace_3mm_standardized_mean, 'inputnode.mask_file') wf.connect(falff_MNIspace_3mm_standardized_mean, 'outputnode.out_file', ds, 'alff.falff_MNI_3mm_standardized_mean') # REHO reho = cpac_reho.create_reho() reho.inputs.inputspec.cluster_size = 27 wf.connect(selectfiles, 'preproc_epi_bp', reho, 'inputspec.rest_res_filt') # wf.connect(GM_mask_epiSpace, 'out_file', reho, 'inputspec.rest_mask') wf.connect(selectfiles, 'epi_mask', reho, 'inputspec.rest_mask') wf.connect(reho, 'outputspec.raw_reho_map', ds, 'reho.reho') # REHO 2 MNI # fixme spline or default? reho_MNIspace_3mm = Node(fsl.ApplyWarp(), name='reho_MNIspace_3mm') reho_MNIspace_3mm.inputs.interp = 'spline' reho_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'} wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', reho_MNIspace_3mm, 'ref_file') wf.connect(reho, 'outputspec.raw_reho_map', reho_MNIspace_3mm, 'in_file') wf.connect(selectfiles, 'epi_2_MNI_warp', reho_MNIspace_3mm, 'field_file') wf.connect(reho_MNIspace_3mm, 'out_file', ds, 'reho.reho_MNI_3mm') # REHO_MNI Z-SCORE reho_MNIspace_3mm_Z = cpac_utils.get_zscore(input_name='reho_MNIspace_3mm', wf_name='reho_MNIspace_3mm_Z') wf.connect(alff_MNIspace_3mm, 'out_file', reho_MNIspace_3mm_Z, 'inputspec.input_file') # wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', reho_MNIspace_3mm_Z, 'inputspec.mask_file') wf.connect(epi_mask_MNIspace_3mm, 'out_file', reho_MNIspace_3mm_Z, 'inputspec.mask_file') wf.connect(reho_MNIspace_3mm_Z, 'outputspec.z_score_img', ds, 'reho.reho_MNI_3mm_Z') # REHO_MNI STANDARDIZE BY MEAN reho_MNIspace_3mm_standardized_mean = calc_metrics_utils.standardize_divide_by_mean( wf_name='reho_MNIspace_3mm_standardized_mean') wf.connect(reho_MNIspace_3mm, 'out_file', reho_MNIspace_3mm_standardized_mean, 'inputnode.in_file') wf.connect(epi_mask_MNIspace_3mm, 'out_file', reho_MNIspace_3mm_standardized_mean, 'inputnode.mask_file') wf.connect(reho_MNIspace_3mm_standardized_mean, 'outputnode.out_file', ds, 'reho.reho_MNI_3mm_standardized_mean') # VMHC # create registration to symmetrical MNI template struct_2_MNI_symm = cpac_registration.create_nonlinear_register(name='struct_2_MNI_symm') wf.connect(selectfiles_anat_templates, 'vmhc_config_file_2mm', struct_2_MNI_symm, 'inputspec.fnirt_config') wf.connect(selectfiles_anat_templates, 'vmhc_symm_brain', struct_2_MNI_symm, 'inputspec.reference_brain') wf.connect(selectfiles_anat_templates, 'vmhc_symm_skull', struct_2_MNI_symm, 'inputspec.reference_skull') wf.connect(selectfiles_anat_templates, 'vmhc_symm_brain_mask_dil', struct_2_MNI_symm, 'inputspec.ref_mask') wf.connect(selectfiles, 't1w', struct_2_MNI_symm, 'inputspec.input_skull') wf.connect(selectfiles, 't1w_brain', struct_2_MNI_symm, 'inputspec.input_brain') wf.connect(struct_2_MNI_symm, 'outputspec.output_brain', ds, 'vmhc.symm_reg.@output_brain') wf.connect(struct_2_MNI_symm, 'outputspec.linear_xfm', ds, 'vmhc.symm_reg.@linear_xfm') wf.connect(struct_2_MNI_symm, 'outputspec.invlinear_xfm', ds, 'vmhc.symm_reg.@invlinear_xfm') wf.connect(struct_2_MNI_symm, 'outputspec.nonlinear_xfm', ds, 'vmhc.symm_reg.@nonlinear_xfm') # fixme vmhc = cpac_vmhc.create_vmhc(use_ants=False, name='vmhc') vmhc.inputs.fwhm_input.fwhm = 4 wf.connect(selectfiles_anat_templates, 'vmhc_symm_brain_3mm', vmhc, 'inputspec.standard_for_func') wf.connect(selectfiles, 'preproc_epi_bp_tNorm', vmhc, 'inputspec.rest_res') wf.connect(selectfiles, 'epi_2_struct_mat', vmhc, 'inputspec.example_func2highres_mat') wf.connect(struct_2_MNI_symm, 'outputspec.nonlinear_xfm', vmhc, 'inputspec.fnirt_nonlinear_warp') # wf.connect(GM_mask_epiSpace, 'out_file', vmhc, 'inputspec.rest_mask') wf.connect(selectfiles, 'epi_mask', vmhc, 'inputspec.rest_mask') wf.connect(vmhc, 'outputspec.rest_res_2symmstandard', ds, 'vmhc.rest_res_2symmstandard') wf.connect(vmhc, 'outputspec.VMHC_FWHM_img', ds, 'vmhc.VMHC_FWHM_img') wf.connect(vmhc, 'outputspec.VMHC_Z_FWHM_img', ds, 'vmhc.VMHC_Z_FWHM_img') wf.connect(vmhc, 'outputspec.VMHC_Z_stat_FWHM_img', ds, 'vmhc.VMHC_Z_stat_FWHM_img') # VARIABILITY SCORES variability = Node(util.Function(input_names=['in_file'], output_names=['out_file_list'], function=calc_metrics_utils.calc_variability), name='variability') wf.connect(selectfiles, 'preproc_epi_bp', variability, 'in_file') wf.connect(variability, 'out_file_list', ds, 'variability.subjectSpace.@out_files') # #fixme spline? variabilty_MNIspace_3mm = MapNode(fsl.ApplyWarp(), iterfield=['in_file'], name='variabilty_MNIspace_3mm') variabilty_MNIspace_3mm.inputs.interp = 'spline' variabilty_MNIspace_3mm.plugin_args = {'submit_specs': 'request_memory = 4000'} wf.connect(selectfiles_anat_templates, 'FSL_MNI_3mm_template', variabilty_MNIspace_3mm, 'ref_file') wf.connect(selectfiles, 'epi_2_MNI_warp', variabilty_MNIspace_3mm, 'field_file') wf.connect(variability, 'out_file_list', variabilty_MNIspace_3mm, 'in_file') wf.connect(variabilty_MNIspace_3mm, 'out_file', ds, 'variability.MNI_3mm.@out_file') # CALC Z SCORE variabilty_MNIspace_3mm_Z = cpac_centrality_z_score.get_cent_zscore(wf_name='variabilty_MNIspace_3mm_Z') wf.connect(variabilty_MNIspace_3mm, 'out_file', variabilty_MNIspace_3mm_Z, 'inputspec.input_file') # wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', variabilty_MNIspace_3mm_Z, 'inputspec.mask_file') wf.connect(epi_mask_MNIspace_3mm, 'out_file', variabilty_MNIspace_3mm_Z, 'inputspec.mask_file') wf.connect(variabilty_MNIspace_3mm_Z, 'outputspec.z_score_img', ds, 'variability.MNI_3mm_Z.@out_file') # STANDARDIZE BY MEAN variabilty_MNIspace_3mm_standardized_mean = calc_metrics_utils.standardize_divide_by_mean( wf_name='variabilty_MNIspace_3mm_standardized_mean') wf.connect(variabilty_MNIspace_3mm, 'out_file', variabilty_MNIspace_3mm_standardized_mean, 'inputnode.in_file') wf.connect(epi_mask_MNIspace_3mm, 'out_file', variabilty_MNIspace_3mm_standardized_mean, 'inputnode.mask_file') wf.connect(variabilty_MNIspace_3mm_standardized_mean, 'outputnode.out_file', ds, 'variability.MNI_3mm_standardized_mean.@out_file') wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def segmentation(projectid, subjectid, sessionid, master_config, onlyT1=True, pipeline_name=''): import os.path import nipype.pipeline.engine as pe import nipype.interfaces.io as nio from nipype.interfaces import ants from nipype.interfaces.utility import IdentityInterface, Function, Merge # Set universal pipeline options from nipype import config config.update_config(master_config) from PipeLineFunctionHelpers import ClipT1ImageWithBrainMask from .WorkupT1T2BRAINSCut import CreateBRAINSCutWorkflow from utilities.distributed import modify_qsub_args from nipype.interfaces.semtools import BRAINSSnapShotWriter # CLUSTER_QUEUE=master_config['queue'] CLUSTER_QUEUE_LONG = master_config['long_q'] baw200 = pe.Workflow(name=pipeline_name) # HACK: print for debugging for key, itme in list(master_config.items()): print(("-" * 30)) print((key, ":", itme)) print(("-" * 30)) # END HACK inputsSpec = pe.Node(interface=IdentityInterface(fields=['t1_average', 't2_average', 'template_t1', 'hncma_atlas', 'LMIatlasToSubject_tx', 'inputLabels', 'inputHeadLabels', 'posteriorImages', 'UpdatedPosteriorsList', 'atlasToSubjectRegistrationState', 'rho', 'phi', 'theta', 'l_caudate_ProbabilityMap', 'r_caudate_ProbabilityMap', 'l_hippocampus_ProbabilityMap', 'r_hippocampus_ProbabilityMap', 'l_putamen_ProbabilityMap', 'r_putamen_ProbabilityMap', 'l_thalamus_ProbabilityMap', 'r_thalamus_ProbabilityMap', 'l_accumben_ProbabilityMap', 'r_accumben_ProbabilityMap', 'l_globus_ProbabilityMap', 'r_globus_ProbabilityMap', 'trainModelFile_txtD0060NT0060_gz', ]), run_without_submitting=True, name='inputspec') # outputsSpec = pe.Node(interface=IdentityInterface(fields=[...]), # run_without_submitting=True, name='outputspec') currentClipT1ImageWithBrainMaskName = 'ClipT1ImageWithBrainMask_' + str(subjectid) + "_" + str(sessionid) ClipT1ImageWithBrainMaskNode = pe.Node(interface=Function(function=ClipT1ImageWithBrainMask, input_names=['t1_image', 'brain_labels', 'clipped_file_name'], output_names=['clipped_file']), name=currentClipT1ImageWithBrainMaskName) ClipT1ImageWithBrainMaskNode.inputs.clipped_file_name = 'clipped_from_BABC_labels_t1.nii.gz' baw200.connect([(inputsSpec, ClipT1ImageWithBrainMaskNode, [('t1_average', 't1_image'), ('inputLabels', 'brain_labels')])]) currentA2SantsRegistrationPostABCSyN = 'A2SantsRegistrationPostABCSyN_' + str(subjectid) + "_" + str(sessionid) ## TODO: It would be great to update the BRAINSABC atlasToSubjectTransform at this point, but ## That requires more testing, and fixes to ANTS to properly collapse transforms. ## For now we are simply creating a dummy node to pass through A2SantsRegistrationPostABCSyN = pe.Node(interface=ants.Registration(), name=currentA2SantsRegistrationPostABCSyN) many_cpu_ANTsSyN_options_dictionary = {'qsub_args': modify_qsub_args(CLUSTER_QUEUE_LONG, 8, 8, 16), 'overwrite': True} A2SantsRegistrationPostABCSyN.plugin_args = many_cpu_ANTsSyN_options_dictionary CommonANTsRegistrationSettings( antsRegistrationNode=A2SantsRegistrationPostABCSyN, registrationTypeDescription="A2SantsRegistrationPostABCSyN", output_transform_prefix='AtlasToSubjectPostBABC_SyN', output_warped_image='atlas2subjectPostBABC.nii.gz', output_inverse_warped_image='subject2atlasPostBABC.nii.gz', save_state='SavedInternalSyNStatePostBABC.h5', invert_initial_moving_transform=False, initial_moving_transform=None) ## TODO: Try multi-modal registration here baw200.connect([(inputsSpec, A2SantsRegistrationPostABCSyN, [('atlasToSubjectRegistrationState', 'restore_state'), ('t1_average', 'fixed_image'), ('template_t1', 'moving_image')]) ]) myLocalSegWF = CreateBRAINSCutWorkflow(projectid, subjectid, sessionid, master_config['queue'], master_config['long_q'], "Segmentation", onlyT1) MergeStage2AverageImagesName = "99_mergeAvergeStage2Images_" + str(sessionid) MergeStage2AverageImages = pe.Node(interface=Merge(2), run_without_submitting=True, name=MergeStage2AverageImagesName) baw200.connect([(inputsSpec, myLocalSegWF, [('t1_average', 'inputspec.T1Volume'), ('template_t1', 'inputspec.template_t1'), ('posteriorImages', "inputspec.posteriorDictionary"), ('inputLabels', 'inputspec.RegistrationROI'), ]), (inputsSpec, MergeStage2AverageImages, [('t1_average', 'in1')]), (A2SantsRegistrationPostABCSyN, myLocalSegWF, [('composite_transform', 'inputspec.atlasToSubjectTransform')]) ]) baw200.connect([(inputsSpec, myLocalSegWF, [ ('rho', 'inputspec.rho'), ('phi', 'inputspec.phi'), ('theta', 'inputspec.theta'), ('l_caudate_ProbabilityMap', 'inputspec.l_caudate_ProbabilityMap'), ('r_caudate_ProbabilityMap', 'inputspec.r_caudate_ProbabilityMap'), ('l_hippocampus_ProbabilityMap', 'inputspec.l_hippocampus_ProbabilityMap'), ('r_hippocampus_ProbabilityMap', 'inputspec.r_hippocampus_ProbabilityMap'), ('l_putamen_ProbabilityMap', 'inputspec.l_putamen_ProbabilityMap'), ('r_putamen_ProbabilityMap', 'inputspec.r_putamen_ProbabilityMap'), ('l_thalamus_ProbabilityMap', 'inputspec.l_thalamus_ProbabilityMap'), ('r_thalamus_ProbabilityMap', 'inputspec.r_thalamus_ProbabilityMap'), ('l_accumben_ProbabilityMap', 'inputspec.l_accumben_ProbabilityMap'), ('r_accumben_ProbabilityMap', 'inputspec.r_accumben_ProbabilityMap'), ('l_globus_ProbabilityMap', 'inputspec.l_globus_ProbabilityMap'), ('r_globus_ProbabilityMap', 'inputspec.r_globus_ProbabilityMap'), ('trainModelFile_txtD0060NT0060_gz', 'inputspec.trainModelFile_txtD0060NT0060_gz') ] )] ) if not onlyT1: baw200.connect([(inputsSpec, myLocalSegWF, [('t2_average', 'inputspec.T2Volume')]), (inputsSpec, MergeStage2AverageImages, [('t2_average', 'in2')])]) file_count = 15 # Count of files to merge into MergeSessionSubjectToAtlas else: file_count = 14 # Count of files to merge into MergeSessionSubjectToAtlas ## NOTE: Element 0 of AccumulatePriorsList is the accumulated GM tissue # baw200.connect([(AccumulateLikeTissuePosteriorsNode, myLocalSegWF, # [(('AccumulatePriorsList', getListIndex, 0), "inputspec.TotalGM")]), # ]) ### Now define where the final organized outputs should go. DataSink = pe.Node(nio.DataSink(), name="CleanedDenoisedSegmentation_DS_" + str(subjectid) + "_" + str(sessionid)) DataSink.overwrite = master_config['ds_overwrite'] DataSink.inputs.base_directory = master_config['resultdir'] # DataSink.inputs.regexp_substitutions = GenerateOutputPattern(projectid, subjectid, sessionid,'BRAINSCut') # DataSink.inputs.regexp_substitutions = GenerateBRAINSCutImagesOutputPattern(projectid, subjectid, sessionid) DataSink.inputs.substitutions = [ ('Segmentations', os.path.join(projectid, subjectid, sessionid, 'CleanedDenoisedRFSegmentations')), ('subjectANNLabel_', ''), ('ANNContinuousPrediction', ''), ('subject.nii.gz', '.nii.gz'), ('_seg.nii.gz', '_seg.nii.gz'), ('.nii.gz', '_seg.nii.gz'), ('_seg_seg', '_seg')] baw200.connect([(myLocalSegWF, DataSink, [('outputspec.outputBinaryLeftCaudate', 'Segmentations.@LeftCaudate'), ('outputspec.outputBinaryRightCaudate', 'Segmentations.@RightCaudate'), ('outputspec.outputBinaryLeftHippocampus', 'Segmentations.@LeftHippocampus'), ('outputspec.outputBinaryRightHippocampus', 'Segmentations.@RightHippocampus'), ('outputspec.outputBinaryLeftPutamen', 'Segmentations.@LeftPutamen'), ('outputspec.outputBinaryRightPutamen', 'Segmentations.@RightPutamen'), ('outputspec.outputBinaryLeftThalamus', 'Segmentations.@LeftThalamus'), ('outputspec.outputBinaryRightThalamus', 'Segmentations.@RightThalamus'), ('outputspec.outputBinaryLeftAccumben', 'Segmentations.@LeftAccumben'), ('outputspec.outputBinaryRightAccumben', 'Segmentations.@RightAccumben'), ('outputspec.outputBinaryLeftGlobus', 'Segmentations.@LeftGlobus'), ('outputspec.outputBinaryRightGlobus', 'Segmentations.@RightGlobus'), ('outputspec.outputLabelImageName', 'Segmentations.@LabelImageName'), ('outputspec.outputCSVFileName', 'Segmentations.@CSVFileName')]), # (myLocalSegWF, DataSink, [('outputspec.cleaned_labels', 'Segmentations.@cleaned_labels')]) ]) MergeStage2BinaryVolumesName = "99_MergeStage2BinaryVolumes_" + str(sessionid) MergeStage2BinaryVolumes = pe.Node(interface=Merge(12), run_without_submitting=True, name=MergeStage2BinaryVolumesName) baw200.connect([(myLocalSegWF, MergeStage2BinaryVolumes, [('outputspec.outputBinaryLeftAccumben', 'in1'), ('outputspec.outputBinaryLeftCaudate', 'in2'), ('outputspec.outputBinaryLeftPutamen', 'in3'), ('outputspec.outputBinaryLeftGlobus', 'in4'), ('outputspec.outputBinaryLeftThalamus', 'in5'), ('outputspec.outputBinaryLeftHippocampus', 'in6'), ('outputspec.outputBinaryRightAccumben', 'in7'), ('outputspec.outputBinaryRightCaudate', 'in8'), ('outputspec.outputBinaryRightPutamen', 'in9'), ('outputspec.outputBinaryRightGlobus', 'in10'), ('outputspec.outputBinaryRightThalamus', 'in11'), ('outputspec.outputBinaryRightHippocampus', 'in12')]) ]) ## SnapShotWriter for Segmented result checking: SnapShotWriterNodeName = "SnapShotWriter_" + str(sessionid) SnapShotWriter = pe.Node(interface=BRAINSSnapShotWriter(), name=SnapShotWriterNodeName) SnapShotWriter.inputs.outputFilename = 'snapShot' + str(sessionid) + '.png' # output specification SnapShotWriter.inputs.inputPlaneDirection = [2, 1, 1, 1, 1, 0, 0] SnapShotWriter.inputs.inputSliceToExtractInPhysicalPoint = [-3, -7, -3, 5, 7, 22, -22] baw200.connect([(MergeStage2AverageImages, SnapShotWriter, [('out', 'inputVolumes')]), (MergeStage2BinaryVolumes, SnapShotWriter, [('out', 'inputBinaryVolumes')]), (SnapShotWriter, DataSink, [('outputFilename', 'Segmentations.@outputSnapShot')]) ]) # currentAntsLabelWarpToSubject = 'AntsLabelWarpToSubject' + str(subjectid) + "_" + str(sessionid) # AntsLabelWarpToSubject = pe.Node(interface=ants.ApplyTransforms(), name=currentAntsLabelWarpToSubject) # # AntsLabelWarpToSubject.inputs.num_threads = -1 # AntsLabelWarpToSubject.inputs.dimension = 3 # AntsLabelWarpToSubject.inputs.output_image = 'warped_hncma_atlas_seg.nii.gz' # AntsLabelWarpToSubject.inputs.interpolation = "MultiLabel" # # baw200.connect([(A2SantsRegistrationPostABCSyN, AntsLabelWarpToSubject, [('composite_transform', 'transforms')]), # (inputsSpec, AntsLabelWarpToSubject, [('t1_average', 'reference_image'), # ('hncma_atlas', 'input_image')]) # ]) # ##### # ### Now define where the final organized outputs should go. # AntsLabelWarpedToSubject_DSName = "AntsLabelWarpedToSubject_DS_" + str(sessionid) # AntsLabelWarpedToSubject_DS = pe.Node(nio.DataSink(), name=AntsLabelWarpedToSubject_DSName) # AntsLabelWarpedToSubject_DS.overwrite = master_config['ds_overwrite'] # AntsLabelWarpedToSubject_DS.inputs.base_directory = master_config['resultdir'] # AntsLabelWarpedToSubject_DS.inputs.substitutions = [('AntsLabelWarpedToSubject', os.path.join(projectid, subjectid, sessionid, 'AntsLabelWarpedToSubject'))] # # baw200.connect([(AntsLabelWarpToSubject, AntsLabelWarpedToSubject_DS, [('output_image', 'AntsLabelWarpedToSubject')])]) MergeSessionSubjectToAtlasName = "99_MergeSessionSubjectToAtlas_" + str(sessionid) MergeSessionSubjectToAtlas = pe.Node(interface=Merge(file_count), run_without_submitting=True, name=MergeSessionSubjectToAtlasName) baw200.connect([(myLocalSegWF, MergeSessionSubjectToAtlas, [('outputspec.outputBinaryLeftAccumben', 'in1'), ('outputspec.outputBinaryLeftCaudate', 'in2'), ('outputspec.outputBinaryLeftPutamen', 'in3'), ('outputspec.outputBinaryLeftGlobus', 'in4'), ('outputspec.outputBinaryLeftThalamus', 'in5'), ('outputspec.outputBinaryLeftHippocampus', 'in6'), ('outputspec.outputBinaryRightAccumben', 'in7'), ('outputspec.outputBinaryRightCaudate', 'in8'), ('outputspec.outputBinaryRightPutamen', 'in9'), ('outputspec.outputBinaryRightGlobus', 'in10'), ('outputspec.outputBinaryRightThalamus', 'in11'), ('outputspec.outputBinaryRightHippocampus', 'in12')]), # (FixWMPartitioningNode, MergeSessionSubjectToAtlas, [('UpdatedPosteriorsList', 'in13')]), (inputsSpec, MergeSessionSubjectToAtlas, [('UpdatedPosteriorsList', 'in13')]), (inputsSpec, MergeSessionSubjectToAtlas, [('t1_average', 'in14')]) ]) if not onlyT1: assert file_count == 15 baw200.connect([(inputsSpec, MergeSessionSubjectToAtlas, [('t2_average', 'in15')])]) LinearSubjectToAtlasANTsApplyTransformsName = 'LinearSubjectToAtlasANTsApplyTransforms_' + str(sessionid) LinearSubjectToAtlasANTsApplyTransforms = pe.MapNode(interface=ants.ApplyTransforms(), iterfield=['input_image'], name=LinearSubjectToAtlasANTsApplyTransformsName) LinearSubjectToAtlasANTsApplyTransforms.inputs.num_threads = -1 LinearSubjectToAtlasANTsApplyTransforms.inputs.interpolation = 'Linear' baw200.connect( [(A2SantsRegistrationPostABCSyN, LinearSubjectToAtlasANTsApplyTransforms, [('inverse_composite_transform', 'transforms')]), (inputsSpec, LinearSubjectToAtlasANTsApplyTransforms, [('template_t1', 'reference_image')]), (MergeSessionSubjectToAtlas, LinearSubjectToAtlasANTsApplyTransforms, [('out', 'input_image')]) ]) MergeMultiLabelSessionSubjectToAtlasName = "99_MergeMultiLabelSessionSubjectToAtlas_" + str(sessionid) MergeMultiLabelSessionSubjectToAtlas = pe.Node(interface=Merge(2), run_without_submitting=True, name=MergeMultiLabelSessionSubjectToAtlasName) baw200.connect([(inputsSpec, MergeMultiLabelSessionSubjectToAtlas, [('inputLabels', 'in1'), ('inputHeadLabels', 'in2')]) ]) ### This is taking this sessions RF label map back into NAC atlas space. # { MultiLabelSubjectToAtlasANTsApplyTransformsName = 'MultiLabelSubjectToAtlasANTsApplyTransforms_' + str( sessionid) + '_map' MultiLabelSubjectToAtlasANTsApplyTransforms = pe.MapNode(interface=ants.ApplyTransforms(), iterfield=['input_image'], name=MultiLabelSubjectToAtlasANTsApplyTransformsName) MultiLabelSubjectToAtlasANTsApplyTransforms.inputs.num_threads = -1 MultiLabelSubjectToAtlasANTsApplyTransforms.inputs.interpolation = 'MultiLabel' baw200.connect([(A2SantsRegistrationPostABCSyN, MultiLabelSubjectToAtlasANTsApplyTransforms, [('inverse_composite_transform', 'transforms')]), (inputsSpec, MultiLabelSubjectToAtlasANTsApplyTransforms, [('template_t1', 'reference_image')]), (MergeMultiLabelSessionSubjectToAtlas, MultiLabelSubjectToAtlasANTsApplyTransforms, [('out', 'input_image')]) ]) # } ### Now we must take the sessions to THIS SUBJECTS personalized atlas. # { # } ### Now define where the final organized outputs should go. Subj2Atlas_DSName = "SubjectToAtlas_DS_" + str(sessionid) Subj2Atlas_DS = pe.Node(nio.DataSink(), name=Subj2Atlas_DSName) Subj2Atlas_DS.overwrite = master_config['ds_overwrite'] Subj2Atlas_DS.inputs.base_directory = master_config['resultdir'] Subj2Atlas_DS.inputs.regexp_substitutions = [(r'_LinearSubjectToAtlasANTsApplyTransforms_[^/]*', r'' + sessionid + '/')] baw200.connect([(LinearSubjectToAtlasANTsApplyTransforms, Subj2Atlas_DS, [('output_image', 'SubjectToAtlasWarped.@linear_output_images')])]) Subj2AtlasTransforms_DSName = "SubjectToAtlasTransforms_DS_" + str(sessionid) Subj2AtlasTransforms_DS = pe.Node(nio.DataSink(), name=Subj2AtlasTransforms_DSName) Subj2AtlasTransforms_DS.overwrite = master_config['ds_overwrite'] Subj2AtlasTransforms_DS.inputs.base_directory = master_config['resultdir'] Subj2AtlasTransforms_DS.inputs.regexp_substitutions = [(r'SubjectToAtlasWarped', r'SubjectToAtlasWarped/' + sessionid + '/')] baw200.connect([(A2SantsRegistrationPostABCSyN, Subj2AtlasTransforms_DS, [('composite_transform', 'SubjectToAtlasWarped.@composite_transform'), ('inverse_composite_transform', 'SubjectToAtlasWarped.@inverse_composite_transform')])]) # baw200.connect([(MultiLabelSubjectToAtlasANTsApplyTransforms, Subj2Atlas_DS, [('output_image', 'SubjectToAtlasWarped.@multilabel_output_images')])]) if master_config['plugin_name'].startswith( 'SGE'): # for some nodes, the qsub call needs to be modified on the cluster A2SantsRegistrationPostABCSyN.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], 8, 8, 24)} SnapShotWriter.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], 1, 1, 1)} LinearSubjectToAtlasANTsApplyTransforms.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], 1, 1, 1)} MultiLabelSubjectToAtlasANTsApplyTransforms.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], 1, 1, 1)} return baw200
def main(): """Entry point""" parser = ArgumentParser(description='MRI Quality Control', formatter_class=RawTextHelpFormatter) g_input = parser.add_argument_group('Inputs') g_input.add_argument('-B', '--bids-root', action='store', default=os.getcwd()) g_input.add_argument('-i', '--input-folder', action='store') g_input.add_argument('-S', '--subject-id', nargs='*', action='store') g_input.add_argument('-s', '--session-id', action='store') g_input.add_argument('-r', '--run-id', action='store') g_input.add_argument('-d', '--data-type', action='store', nargs='*', choices=['anat', 'func'], default=['anat', 'func']) g_input.add_argument('-v', '--version', action='store_true', default=False, help='Show current mriqc version') g_input.add_argument('--nthreads', action='store', default=0, type=int, help='number of threads') g_input.add_argument('--write-graph', action='store_true', default=False, help='Write workflow graph.') g_input.add_argument('--test-run', action='store_true', default=False, help='Do not run the workflow.') g_input.add_argument('--use-plugin', action='store', default=None, help='nipype plugin configuration file') g_input.add_argument('--save-memory', action='store_true', default=False, help='Save as much memory as possible') g_input.add_argument('--hmc-afni', action='store_true', default=False, help='Use ANFI 3dvolreg for head motion correction (HMC) and ' 'frame displacement (FD) estimation') g_input.add_argument('--ants-settings', action='store', help='path to JSON file with settings for ANTS') g_outputs = parser.add_argument_group('Outputs') g_outputs.add_argument('-o', '--output-dir', action='store') g_outputs.add_argument('-w', '--work-dir', action='store', default=op.join(os.getcwd(), 'work')) opts = parser.parse_args() bids_root = op.abspath(opts.bids_root) if opts.input_folder is not None: warn('The --input-folder flag is deprecated, please use -B instead', DeprecationWarning) if bids_root == os.getcwd(): bids_root = op.abspath(opts.input_folder) if opts.version: print('mriqc version ' + __version__) exit(0) settings = {'bids_root': bids_root, 'output_dir': os.getcwd(), 'write_graph': opts.write_graph, 'save_memory': opts.save_memory, 'hmc_afni': opts.hmc_afni, 'nthreads': opts.nthreads} if opts.output_dir: settings['output_dir'] = op.abspath(opts.output_dir) if not op.exists(settings['output_dir']): os.makedirs(settings['output_dir']) settings['work_dir'] = op.abspath(opts.work_dir) with LockFile(settings['work_dir']): if not op.exists(settings['work_dir']): os.makedirs(settings['work_dir']) if opts.ants_settings: settings['ants_settings'] = opts.ants_settings log_dir = op.join(settings['work_dir'] + '_log') if not op.exists(log_dir): os.makedirs(log_dir) # Set nipype config ncfg.update_config({ 'logging': {'log_directory': log_dir, 'log_to_file': True}, 'execution': {'crashdump_dir': log_dir} }) plugin_settings = {'plugin': 'Linear'} if opts.use_plugin is not None: from yaml import load as loadyml with open(opts.use_plugin) as pfile: plugin_settings = loadyml(pfile) else: # Setup multiprocessing if settings['nthreads'] == 0: settings['nthreads'] = cpu_count() if settings['nthreads'] > 1: plugin_settings['plugin'] = 'MultiProc' plugin_settings['plugin_args'] = {'n_procs': settings['nthreads']} for dtype in opts.data_type: ms_func = getattr(mwc, 'ms_' + dtype) workflow = ms_func(subject_id=opts.subject_id, session_id=opts.session_id, run_id=opts.run_id, settings=settings) if workflow is None: LOGGER.warn('No {} scans were found in {}', dtype, settings['bids_root']) continue workflow.base_dir = settings['work_dir'] if settings.get('write_graph', False): workflow.write_graph() if not opts.test_run: workflow.run(**plugin_settings) if opts.subject_id is None and not opts.test_run: workflow_report(dtype, settings)
def learning_prepare_data_wf(working_dir, ds_dir, template_dir, df_file, in_data_name_list, data_lookup_dict, use_n_procs, plugin_name): import os from nipype import config from nipype.pipeline.engine import Node, Workflow, MapNode, JoinNode import nipype.interfaces.utility as util import nipype.interfaces.io as nio from nipype.interfaces.freesurfer.utils import ImageInfo from utils import aggregate_data, vectorize_data from itertools import chain # ensure in_data_name_list is list of lists in_data_name_list = [i if type(i) == list else [i] for i in in_data_name_list] in_data_name_list_unique = list(set(chain.from_iterable(in_data_name_list))) ##################################### # GENERAL SETTINGS ##################################### wf = Workflow(name='learning_prepare_data_wf') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': True, 'remove_unnecessary_outputs': False, 'job_finished_timeout': 120}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash') ds = Node(nio.DataSink(), name='ds') ds.inputs.base_directory = os.path.join(ds_dir, 'group_learning_prepare_data') ds.inputs.regexp_substitutions = [ # ('subject_id_', ''), ('_parcellation_', ''), ('_bp_freqs_', 'bp_'), ('_extraction_method_', ''), ('_subject_id_[A0-9]*/', '') ] ds_pdf = Node(nio.DataSink(), name='ds_pdf') ds_pdf.inputs.base_directory = os.path.join(ds_dir, 'pdfs') ds_pdf.inputs.parameterization = False # get atlas data templates_atlases = {'GM_mask_MNI_2mm': 'SPM_GM/SPM_GM_mask_2mm.nii.gz', 'GM_mask_MNI_3mm': 'SPM_GM/SPM_GM_mask_3mm.nii.gz', 'brain_mask_MNI_3mm': 'cpac_image_resources/MNI_3mm/MNI152_T1_3mm_brain_mask.nii.gz', 'brain_template_MNI_3mm': 'cpac_image_resources/MNI_3mm/MNI152_T1_3mm.nii.gz' } selectfiles_anat_templates = Node(nio.SelectFiles(templates_atlases, base_directory=template_dir), name="selectfiles_anat_templates") ##################################### # SET ITERATORS ##################################### # SUBJECTS ITERATOR in_data_name_infosource = Node(util.IdentityInterface(fields=['in_data_name']), name='in_data_name_infosource') in_data_name_infosource.iterables = ('in_data_name', in_data_name_list_unique) mulitmodal_in_data_name_infosource = Node(util.IdentityInterface(fields=['multimodal_in_data_name']), name='mulitmodal_in_data_name_infosource') mulitmodal_in_data_name_infosource.iterables = ('multimodal_in_data_name', in_data_name_list) subjects_selection_crit_dict = {} subjects_selection_crit_dict['adult_healthy_F'] = ["df[df.sex == \'F\']", 'df[df.no_axis_1]', 'df[df.age >= 18]'] subjects_selection_crit_dict['adult_F'] = ["df[df.sex == \'F\']", 'df[df.age >= 18]'] subjects_selection_crit_dict['F'] = ["df[df.sex == \'F\']"] subjects_selection_crit_dict['adult_healthy_M'] = ["df[df.sex == \'M\']", 'df[df.no_axis_1]', 'df[df.age >= 18]'] subjects_selection_crit_dict['adult_M'] = ["df[df.sex == \'M\']", 'df[df.age >= 18]'] subjects_selection_crit_dict['adult'] = ['df[df.age >= 18]'] # subjects_selection_crit_names_list = subjects_selection_crit_dict.keys() subjects_selection_crit_names_list = ['adult_F'] subject_selection_infosource = Node(util.IdentityInterface(fields=['selection_criterium']), name='subject_selection_infosource') subject_selection_infosource.iterables = ('selection_criterium', subjects_selection_crit_names_list) def out_name_str_fct(selection_criterium, in_data_name): return selection_criterium + '_' + in_data_name out_name_str = Node(util.Function(input_names=['selection_criterium', 'in_data_name'], output_names=['out_name_str'], function=out_name_str_fct), name='out_name_str') wf.connect(in_data_name_infosource, 'in_data_name', out_name_str, 'in_data_name') wf.connect(subject_selection_infosource, 'selection_criterium', out_name_str, 'selection_criterium') def get_subjects_info_fct(df_file, subjects_selection_crit_dict, selection_criterium): import pandas as pd import os import numpy as np df = pd.read_pickle(df_file) # EXCLUSION HERE: for eval_str in subjects_selection_crit_dict[selection_criterium]: df = eval(eval_str) df_out_file = os.path.join(os.getcwd(), 'df_use.csv') df.to_csv(df_out_file) subjects_list = df.leica_id.values age = df.age.values age_file = os.path.join(os.getcwd(), 'age.npy') np.save(age_file, age) return (age_file, df_out_file, subjects_list) get_subjects_info = Node( util.Function(input_names=['df_file', 'subjects_selection_crit_dict', 'selection_criterium'], output_names=['age_file', 'df_out_file', 'subjects_list'], function=get_subjects_info_fct), name='get_subjects_info') get_subjects_info.inputs.df_file = df_file get_subjects_info.inputs.subjects_selection_crit_dict = subjects_selection_crit_dict wf.connect(subject_selection_infosource, 'selection_criterium', get_subjects_info, 'selection_criterium') wf.connect(get_subjects_info, 'df_out_file', ds, 'test') wf.connect(get_subjects_info, 'age_file', ds, 'test_age_file') def create_file_list_fct(subjects_list, in_data_name, data_lookup_dict, brain_mask_path, gm_mask_path): file_list = [] for s in subjects_list: file_list.append(data_lookup_dict[in_data_name]['path_str'].format(subject_id=s)) if 'matrix_name' in data_lookup_dict[in_data_name].keys(): matrix_name = data_lookup_dict[in_data_name]['matrix_name'] else: matrix_name = None if 'parcellation_path' in data_lookup_dict[in_data_name].keys(): parcellation_path = data_lookup_dict[in_data_name]['parcellation_path'] else: parcellation_path = None if 'fwhm' in data_lookup_dict[in_data_name].keys(): fwhm = data_lookup_dict[in_data_name]['fwhm'] else: fwhm = None mask_path = brain_mask_path if 'use_gm_mask' in data_lookup_dict[in_data_name].keys(): if data_lookup_dict[in_data_name]['use_gm_mask']: mask_path = gm_mask_path return file_list, matrix_name, parcellation_path, fwhm, mask_path create_file_list = Node(util.Function(input_names=['subjects_list', 'in_data_name', 'data_lookup_dict', 'brain_mask_path', 'gm_mask_path'], output_names=['file_list', 'matrix_name', 'parcellation_path', 'fwhm', 'mask_path'], function=create_file_list_fct), name='create_file_list') wf.connect(get_subjects_info, 'subjects_list', create_file_list, 'subjects_list') wf.connect(in_data_name_infosource, 'in_data_name', create_file_list, 'in_data_name') create_file_list.inputs.data_lookup_dict = data_lookup_dict wf.connect(selectfiles_anat_templates, 'brain_mask_MNI_3mm', create_file_list, 'brain_mask_path') wf.connect(selectfiles_anat_templates, 'GM_mask_MNI_3mm', create_file_list, 'gm_mask_path') aggregate_subjects = Node(util.Function(input_names=['file_list'], output_names=['merged_file'], function=aggregate_data), name='aggregate_subjects') wf.connect(create_file_list, 'file_list', aggregate_subjects, 'file_list') vectorized_data = Node( util.Function(input_names=['in_data_file', 'mask_file', 'matrix_name', 'parcellation_path', 'fwhm'], output_names=['vectorized_data', 'vectorized_data_file'], function=vectorize_data), name='vectorized_data') wf.connect(aggregate_subjects, 'merged_file', vectorized_data, 'in_data_file') wf.connect(create_file_list, 'mask_path', vectorized_data, 'mask_file') wf.connect(create_file_list, 'matrix_name', vectorized_data, 'matrix_name') wf.connect(create_file_list, 'parcellation_path', vectorized_data, 'parcellation_path') wf.connect(create_file_list, 'fwhm', vectorized_data, 'fwhm') def aggregate_multimodal_metrics_fct(multimodal_list, vectorized_data_file, vectorized_data_names, selection_criterium): import numpy as np import os metrics_index_list = [vectorized_data_names.index(m) for m in multimodal_list] X = None for i in metrics_index_list: X_file = vectorized_data_file[i] X_single = np.load(X_file) if X is None: X = X_single else: X = np.hstack((X, X_single)) multimodal_in_name = '_'.join(multimodal_list) multimodal_out_name = selection_criterium + '_' + multimodal_in_name X_file = os.path.join(os.getcwd(), multimodal_in_name + '.npy') np.save(X_file, X) return multimodal_in_name, multimodal_out_name, X_file aggregate_multimodal_metrics = JoinNode(util.Function(input_names=['multimodal_list', 'vectorized_data_file', 'vectorized_data_names', 'selection_criterium'], output_names=['multimodal_in_name', 'multimodal_out_name', 'X_file'], function=aggregate_multimodal_metrics_fct), joinfield='vectorized_data_file', joinsource=in_data_name_infosource, name='aggregate_multimodal_metrics') wf.connect(mulitmodal_in_data_name_infosource, 'multimodal_in_data_name', aggregate_multimodal_metrics, 'multimodal_list') wf.connect(vectorized_data, 'vectorized_data_file', aggregate_multimodal_metrics, 'vectorized_data_file') wf.connect(subject_selection_infosource, 'selection_criterium', aggregate_multimodal_metrics, 'selection_criterium') wf.connect(aggregate_multimodal_metrics, 'X_file', ds, 'multimodal_test') aggregate_multimodal_metrics.inputs.vectorized_data_names = in_data_name_list_unique def run_prediction(reg_model, X_file, y_file, data_str): def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name): import os import pylab as plt from matplotlib.backends.backend_pdf import PdfPages plt.scatter(y_test, y_test_predicted) plt.plot([10, 80], [10, 80], 'k') plt.xlabel('real') plt.ylabel('predicted') ax = plt.gca() ax.set_aspect('equal') plt.title(title_str) plt.tight_layout() scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf') pp = PdfPages(scatter_file) pp.savefig() pp.close() return scatter_file import os, pickle import numpy as np from sklearn.svm import SVR from sklearn.cross_validation import cross_val_score, cross_val_predict from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import Imputer from sklearn.pipeline import Pipeline from sklearn.metrics import r2_score, mean_absolute_error X = np.load(X_file) y = np.load(y_file) # fixme pipe fill_missing = Imputer() # X = fill_missing.fit_transform(X) # fixme? pipe # remove low variance features var_thr = VarianceThreshold() # X = var_thr.fit_transform(X) # fixme pipe normalize normalize = MinMaxScaler() # X = normalize.fit_transform(X) regression_model = reg_model # SVR(kernel='linear') pipe = Pipeline([ ('fill_missing', fill_missing), ('var_thr', var_thr), ('normalize', normalize), ('regression_model', regression_model), ]) # cv_scores = cross_val_score(pipe, X, y, cv=5, n_jobs=5, # scoring='mean_absolute_error') # cv_scores_r2 = cross_val_score(pipe, X, y, cv=5, n_jobs=5, # scoring='r2') y_predicted = cross_val_predict(pipe, X, y, cv=5, n_jobs=5) cv_scores = mean_absolute_error(y, y_predicted) cv_scores_r2 = r2_score(y, y_predicted) title_str = '{}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}'.format( data_str, cv_scores.mean(), cv_scores.std(), cv_scores.min(), cv_scores.max(), cv_scores_r2.mean(), cv_scores_r2.std(), cv_scores_r2.min(), cv_scores_r2.max()) scatter_file = _pred_real_scatter(y, y_predicted, title_str, data_str) out_file = os.path.join(os.getcwd(), 'cv_scores.pkl') with open(out_file, 'w') as f: pickle.dump(cv_scores, f) y_pred_file = os.path.join(os.getcwd(), 'y_pred.npy') np.save(y_pred_file, y_predicted) return out_file, regression_model, scatter_file, y_pred_file # from sklearn.svm import SVR from sklearn.linear_model import (LassoCV, ElasticNetCV) prediction = Node(util.Function(input_names=['reg_model', 'X_file', 'y_file', 'data_str'], output_names=['out_file', 'regression_model', 'scatter_file', 'y_pred_file'], function=run_prediction), name='prediction') # prediction.inputs.reg_model = SVR(kernel='linear') # wf.connect(aggregate_multimodal_metrics, 'X_file', prediction, 'X_file') # # wf.connect(vectorized_data, 'vectorized_data_file', prediction, 'X_file') # wf.connect(get_subjects_info, 'age_file', prediction, 'y_file') # wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction, 'data_str') # wf.connect(prediction, 'out_file', ds, 'test_naiv_cv_score') # wf.connect(prediction, 'scatter_file', ds_pdf, 'test_naive_scatter_svr') # wf.connect(prediction, 'y_pred_file', ds_pdf, 'test_naive_predicted_svr') # # prediction_lasso = prediction.clone('prediction_lasso') # prediction_lasso.inputs.reg_model = LassoCV(n_jobs=5) # wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_lasso, 'X_file') # wf.connect(get_subjects_info, 'age_file', prediction_lasso, 'y_file') # wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_lasso, 'data_str') # wf.connect(prediction_lasso, 'out_file', ds, 'test_cv_score_lasso') # wf.connect(prediction_lasso, 'scatter_file', ds_pdf, 'test_naive_scatter_lasso') # def run_prediction_split(reg_model, X_file, y_file, data_str): def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name): import os import pylab as plt from matplotlib.backends.backend_pdf import PdfPages plt.scatter(y_test, y_test_predicted) plt.plot([10, 80], [10, 80], 'k') plt.xlabel('real') plt.ylabel('predicted') ax = plt.gca() ax.set_aspect('equal') plt.title(title_str) plt.tight_layout() scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf') pp = PdfPages(scatter_file) pp.savefig() pp.close() return scatter_file import os, pickle import numpy as np from sklearn.svm import SVR from sklearn.cross_validation import cross_val_score, cross_val_predict from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import Imputer from sklearn.pipeline import Pipeline from sklearn.metrics import r2_score, mean_absolute_error from sklearn.cross_validation import train_test_split X = np.load(X_file) y = np.load(y_file) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) fill_missing = Imputer() var_thr = VarianceThreshold() normalize = MinMaxScaler() regression_model = reg_model # SVR(kernel='linear') pipe = Pipeline([ ('fill_missing', fill_missing), ('var_thr', var_thr), ('normalize', normalize), ('regression_model', regression_model), ]) pipe.fit(X_train, y_train) fitted_enet_model = pipe.steps[3][1] results_str = 'n_iter: %s. l1_r: %s. alpha: %s' % ( fitted_enet_model.n_iter_, fitted_enet_model.l1_ratio_, fitted_enet_model.alpha_) y_predicted = pipe.predict(X_test) cv_scores = mean_absolute_error(y_test, y_predicted) cv_scores_r2 = r2_score(y_test, y_predicted) title_str = '{}\n{}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}'.format( data_str, results_str, cv_scores.mean(), cv_scores.std(), cv_scores.min(), cv_scores.max(), cv_scores_r2.mean(), cv_scores_r2.std(), cv_scores_r2.min(), cv_scores_r2.max()) scatter_file = _pred_real_scatter(y_test, y_predicted, title_str, data_str) out_file = os.path.join(os.getcwd(), 'cv_scores.pkl') with open(out_file, 'w') as f: pickle.dump(cv_scores, f) model_out_file = os.path.join(os.getcwd(), 'trained_model.pkl') with open(model_out_file, 'w') as f: pickle.dump(pipe, f) y_pred_file = os.path.join(os.getcwd(), 'y_pred.npy') np.save(y_pred_file, y_predicted) return out_file, regression_model, scatter_file, y_pred_file, model_out_file, fitted_enet_model prediction_enet = Node(util.Function(input_names=['reg_model', 'X_file', 'y_file', 'data_str'], output_names=['out_file', 'regression_model', 'scatter_file', 'y_pred_file', 'model_out_file', 'fitted_enet_model'], function=run_prediction_split), name='prediction_enet') # fixme n_jobs prediction_enet.inputs.reg_model = ElasticNetCV(cv=5, n_jobs=10, selection='random', max_iter=2000, l1_ratio=[0, .01, .05, .1, .5, .7, .9, .95, .99, 1]) wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_enet, 'X_file') wf.connect(get_subjects_info, 'age_file', prediction_enet, 'y_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_enet, 'data_str') wf.connect(prediction_enet, 'out_file', ds, 'test_split_cv_score_enet') wf.connect(prediction_enet, 'model_out_file', ds, 'test_split_trained_model_enet') wf.connect(prediction_enet, 'scatter_file', ds_pdf, 'test_split_scatter_enet') wf.connect(prediction_enet, 'y_pred_file', ds_pdf, 'test_split_predicted_enet') # prediction_enet = prediction.clone('prediction_enet') # prediction_enet.inputs.reg_model = ElasticNetCV(cv=5, n_jobs=7, selection='random', max_iter=2000, # l1_ratio=[.1, .5, .7, .9, .95, .99, 1]) # wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_enet, 'X_file') # wf.connect(get_subjects_info, 'age_file', prediction_enet, 'y_file') # wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_enet, 'data_str') # wf.connect(prediction_enet, 'out_file', ds, 'test_cv_score_enet') # wf.connect(prediction_enet, 'scatter_file', ds_pdf, 'test_naive_scatter_enet') # wf.connect(prediction_enet, 'y_pred_file', ds_pdf, 'test_naive_predicted_enet') # # # # # # # # # # # # # # # GS # # # # # # # # # # # # # # def run_prediction_gs(X_file, y_file, data_str): # fixme copied function def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name): import os import pylab as plt from matplotlib.backends.backend_pdf import PdfPages plt.scatter(y_test, y_test_predicted) plt.plot([10, 80], [10, 80], 'k') plt.xlabel('real') plt.ylabel('predicted') ax = plt.gca() ax.set_aspect('equal') plt.title(title_str) plt.tight_layout() scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf') pp = PdfPages(scatter_file) pp.savefig() pp.close() return scatter_file import os, pickle import numpy as np from sklearn.svm import SVR from sklearn.cross_validation import cross_val_score, cross_val_predict from sklearn.feature_selection import VarianceThreshold from sklearn.grid_search import GridSearchCV from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import Imputer X = np.load(X_file) y = np.load(y_file) # fixme pipe imp = Imputer() X = imp.fit_transform(X) # fixme? pipe # remove low variance features var_thr = VarianceThreshold() X = var_thr.fit_transform(X) # fixme pipe normalize normalize = MinMaxScaler() X = normalize.fit_transform(X) # fixme? # remove low variance features var_thr = VarianceThreshold() X = var_thr.fit_transform(X) regression_model = SVR(kernel='linear') # GRID SEARCH params = { 'C': [.0001, .001, .01, .1, 1, 10, 20, 30], 'epsilon': [.0001, .001, .01, .1, 1], } gs = GridSearchCV(regression_model, params, cv=5, scoring='mean_absolute_error', n_jobs=10) gs.fit(X, y) sorted_grid_score = sorted(gs.grid_scores_, key=lambda x: x.mean_validation_score, reverse=True) score_str = [str(n) + ': ' + str(g) for n, g in enumerate(sorted_grid_score)] gs_file = os.path.join(os.getcwd(), 'gs_' + data_str + '.txt') with open(gs_file, 'w') as f: f.write('\n'.join(score_str)) # GS WITH NESTED CV gs_cv = GridSearchCV(regression_model, params, cv=5, scoring='mean_absolute_error', n_jobs=5) cv_scores = cross_val_score(regression_model, X, y, cv=5, scoring='mean_absolute_error') cv_scores_r2 = cross_val_score(regression_model, X, y, cv=5, scoring='r2') cv_prediction = cross_val_predict(regression_model, X, y, cv=5) title_str = '{}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}\n mean: {:.3f}, sd: {:.3f}, min: {:.3f}, max: {:.3f}'.format( data_str, cv_scores.mean(), cv_scores.std(), cv_scores.min(), cv_scores.max(), cv_scores_r2.mean(), cv_scores_r2.std(), cv_scores_r2.min(), cv_scores_r2.max()) cv_scatter_file = _pred_real_scatter(y, cv_prediction, title_str, data_str + '_cv_pred') ## # # # # # # # # # BEST PARAMETERS best_params = sorted_grid_score[0].parameters regression_model = SVR(kernel='linear', **best_params) y_predicted = cross_val_predict(regression_model, X, y, cv=5, n_jobs=5) best_params_scatter_file = _pred_real_scatter(y, y_predicted, data_str, data_str) return regression_model, gs_file, best_params_scatter_file, cv_scatter_file gs = Node(util.Function(input_names=['X_file', 'y_file', 'data_str'], output_names=['regression_model', 'gs_file', 'best_params_scatter_file', 'cv_scatter_file'], function=run_prediction_gs), name='gs') # wf.connect(aggregate_multimodal_metrics, 'X_file', gs, 'X_file') # wf.connect(get_subjects_info, 'age_file', gs, 'y_file') # wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', gs, 'data_str') # wf.connect(gs, 'gs_file', ds_pdf, 'test_gs') # wf.connect(gs, 'best_params_scatter_file', ds_pdf, 'test_gs_best_params_scatters') # wf.connect(gs, 'cv_scatter_file', ds_pdf, 'test_gs_cv_scatters') # # # # # # # # # # # # # # # PIPELINE # # # # # # # # # # # # # # def run_prediction_gs_split(X_file, y_file, data_str): # fixme copied function def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name): import os import pylab as plt from matplotlib.backends.backend_pdf import PdfPages plt.scatter(y_test, y_test_predicted) plt.plot([10, 80], [10, 80], 'k') plt.xlabel('real') plt.ylabel('predicted') ax = plt.gca() ax.set_aspect('equal') plt.title(title_str) plt.tight_layout() scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf') pp = PdfPages(scatter_file) pp.savefig() pp.close() return scatter_file import os, pickle import numpy as np from sklearn.svm import SVR from sklearn.cross_validation import cross_val_score, cross_val_predict, train_test_split from sklearn.grid_search import GridSearchCV from sklearn.pipeline import Pipeline from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import MinMaxScaler, StandardScaler from sklearn.preprocessing import Imputer from sklearn.feature_selection import SelectPercentile, f_regression from sklearn.metrics import mean_absolute_error, r2_score from sklearn.svm import LinearSVR from sklearn.decomposition import PCA X = np.load(X_file) y = np.load(y_file) # fixme add squared values to X # X = np.hstack([X, np.square(X)]) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) # remove low variance features fill_missing = Imputer() var_thr = VarianceThreshold() normalize = StandardScaler() # MinMaxScaler() selection = SelectPercentile(f_regression) # regression_model = LinearSVR() #SVR(kernel='linear') from sklearn.svm import NuSVR regression_model = LinearSVR() # NuSVR(kernel='linear') #SVR(kernel='linear') pipe = Pipeline([ ('fill_missing', fill_missing), ('var_thr', var_thr), ('normalize', normalize), ('selection', selection), ('regression_model', regression_model), ]) # GRID SEARCH params = { 'selection__percentile': [70, 75, 80, 85, 90, 95, 99, 100], 'regression_model__C': [1, 98, 106, 110, 130, 150, 170, 200, 14450], # [.0001, .001, .01, .1, 1, 10, 20, 30], 'regression_model__epsilon': [0, .005, .01, .05, .1, 1, 5, 10], 'regression_model__loss': ['epsilon_insensitive', 'squared_epsilon_insensitive'], # 'regression_model__nu': [0.01, .25, .5, .75, .8, .85, .9, .95, .99], } # fixme njobs gs = GridSearchCV(pipe, params, cv=5, scoring='mean_absolute_error', n_jobs=30) gs.fit(X_train, y_train) best_estimator = gs.best_estimator_ grid_scores = gs.grid_scores_ gs_file = os.path.join(os.getcwd(), 'gs_' + data_str + '.pkl') with open(gs_file, 'w') as f: pickle.dump(grid_scores, f) sorted_grid_score = sorted(gs.grid_scores_, key=lambda x: x.mean_validation_score, reverse=True) score_str = [str(n) + ': ' + str(g) for n, g in enumerate(sorted_grid_score)] gs_text_file = os.path.join(os.getcwd(), 'gs_txt_' + data_str + '.txt') with open(gs_text_file, 'w') as f: f.write('\n'.join(score_str)) # fitted_model = gs.steps[-1][1] model_out_file = os.path.join(os.getcwd(), 'trained_model.pkl') with open(model_out_file, 'w') as f: pickle.dump(gs, f) y_predicted = gs.predict(X_test) cv_scores = mean_absolute_error(y_test, y_predicted) cv_scores_r2 = r2_score(y_test, y_predicted) title_str = '{}\n mae: {:.3f}\n r2: {:.3f}'.format( data_str, cv_scores, cv_scores_r2) scatter_file = _pred_real_scatter(y_test, y_predicted, title_str, data_str) return model_out_file, scatter_file, gs_text_file, gs_file, best_estimator prediction_gs_split = Node(util.Function(input_names=['X_file', 'y_file', 'data_str'], output_names=['model_out_file', 'scatter_file', 'gs_text_file', 'gs_file', 'best_estimator'], function=run_prediction_gs_split), name='prediction_gs_split') wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_gs_split, 'X_file') wf.connect(get_subjects_info, 'age_file', prediction_gs_split, 'y_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_gs_split, 'data_str') wf.connect(prediction_gs_split, 'scatter_file', ds_pdf, 'test_gs_split_scatter') wf.connect(prediction_gs_split, 'model_out_file', ds, 'test_gs_split_trained_model') wf.connect(prediction_gs_split, 'gs_text_file', ds_pdf, 'test_gs_split_gs_params') wf.connect(prediction_gs_split, 'gs_file', ds_pdf, 'grid_scores') # RANDOM CV def run_prediction_random_gs_split(X_file, y_file, data_str): # fixme copied function def _pred_real_scatter(y_test, y_test_predicted, title_str, in_data_name): import os import pylab as plt from matplotlib.backends.backend_pdf import PdfPages plt.scatter(y_test, y_test_predicted) plt.plot([10, 80], [10, 80], 'k') plt.xlabel('real') plt.ylabel('predicted') ax = plt.gca() ax.set_aspect('equal') plt.title(title_str) plt.tight_layout() scatter_file = os.path.join(os.getcwd(), 'scatter_' + in_data_name + '.pdf') pp = PdfPages(scatter_file) pp.savefig() pp.close() return scatter_file import os, pickle import numpy as np from sklearn.svm import SVR from sklearn.cross_validation import cross_val_score, cross_val_predict, train_test_split from sklearn.grid_search import RandomizedSearchCV from sklearn.pipeline import Pipeline from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import MinMaxScaler, StandardScaler from sklearn.preprocessing import Imputer from sklearn.feature_selection import SelectPercentile, f_regression from sklearn.metrics import mean_absolute_error, r2_score from sklearn.svm import LinearSVR from sklearn.decomposition import PCA from scipy.stats import randint as sp_randint from scipy.stats import expon X = np.load(X_file) y = np.load(y_file) # fixme add squared values to X # X = np.hstack([X, np.square(X)]) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) # remove low variance features fill_missing = Imputer() var_thr = VarianceThreshold() normalize = StandardScaler() # MinMaxScaler() selection = SelectPercentile(f_regression) # regression_model = LinearSVR() #SVR(kernel='linear') from sklearn.svm import NuSVR regression_model = LinearSVR() # NuSVR(kernel='linear') #SVR(kernel='linear') pipe = Pipeline([ ('fill_missing', fill_missing), ('var_thr', var_thr), ('normalize', normalize), ('selection', selection), ('regression_model', regression_model), ]) param_dist = { 'selection__percentile': sp_randint(10, 100), 'regression_model__C': expon(scale=100), # sp_randint(.001, 14450), 'regression_model__epsilon': sp_randint(0, 100), 'regression_model__loss': ['epsilon_insensitive', 'squared_epsilon_insensitive'], } # fixme njobs n_iter_search = 400 gs = RandomizedSearchCV(pipe, param_distributions=param_dist, cv=5, scoring='mean_absolute_error', n_jobs=15, n_iter=n_iter_search) gs.fit(X_train, y_train) best_estimator = gs.best_estimator_ grid_scores = gs.grid_scores_ gs_file = os.path.join(os.getcwd(), 'gs_' + data_str + '.pkl') with open(gs_file, 'w') as f: pickle.dump(grid_scores, f) sorted_grid_score = sorted(gs.grid_scores_, key=lambda x: x.mean_validation_score, reverse=True) score_str = [str(n) + ': ' + str(g) for n, g in enumerate(sorted_grid_score)] gs_text_file = os.path.join(os.getcwd(), 'gs_txt_' + data_str + '.txt') with open(gs_text_file, 'w') as f: f.write('\n'.join(score_str)) # fitted_model = gs.steps[-1][1] # fixme pickle crashes model_out_file = '' # model_out_file = os.path.join(os.getcwd(), 'trained_model.pkl') # with open(model_out_file, 'w') as f: # pickle.dump(gs, f) y_predicted = gs.predict(X_test) cv_scores = mean_absolute_error(y_test, y_predicted) cv_scores_r2 = r2_score(y_test, y_predicted) title_str = '{}\n mae: {:.3f}\n r2: {:.3f}'.format( data_str, cv_scores, cv_scores_r2) scatter_file = _pred_real_scatter(y_test, y_predicted, title_str, data_str) return model_out_file, scatter_file, gs_text_file, gs_file, best_estimator prediction_random_gs_split = Node(util.Function(input_names=['X_file', 'y_file', 'data_str'], output_names=['model_out_file', 'scatter_file', 'gs_text_file', 'gs_file', 'best_estimator'], function=run_prediction_random_gs_split), name='prediction_random_gs_split') wf.connect(aggregate_multimodal_metrics, 'X_file', prediction_random_gs_split, 'X_file') wf.connect(get_subjects_info, 'age_file', prediction_random_gs_split, 'y_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', prediction_random_gs_split, 'data_str') wf.connect(prediction_random_gs_split, 'scatter_file', ds_pdf, 'test_random_gs_split_scatter') #wf.connect(prediction_random_gs_split, 'model_out_file', ds, 'test_random_gs_split_trained_model') wf.connect(prediction_random_gs_split, 'gs_text_file', ds_pdf, 'test_random_gs_split_gs_params') wf.connect(prediction_random_gs_split, 'gs_file', ds_pdf, 'random_grid_scores') from learning_curve import learning_curve_fct learning_curve_svr = Node(util.Function(input_names=['X_file', 'y_file', 'out_name', 'best_estimator'], output_names=['curve_file'], function=learning_curve_fct), name='learning_curve_svr') wf.connect(aggregate_multimodal_metrics, 'X_file', learning_curve_svr, 'X_file') wf.connect(get_subjects_info, 'age_file', learning_curve_svr, 'y_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', learning_curve_svr, 'out_name') wf.connect(prediction_gs_split, 'best_estimator', learning_curve_svr, 'best_estimator') # wf.connect(prediction_gs_split, 'model_out_file', learning_curve_svr, 'fitted_model_file') wf.connect(learning_curve_svr, 'curve_file', ds_pdf, 'learning_curve_svr') def _create_best_enet(fitted_enet): from sklearn.linear_model import ElasticNet best_estimator = ElasticNet(selection='random', max_iter=2000, alpha=fitted_enet.alpha_, l1_ratio=fitted_enet.l1_ratio_) return best_estimator best_estimator_enet = Node(util.Function(input_names=['fitted_enet'], output_names=['best_estimator'], function=_create_best_enet), name='best_estimator_enet') wf.connect(prediction_enet, 'fitted_enet_model', best_estimator_enet, 'fitted_enet') learning_curve_enet = Node(util.Function(input_names=['X_file', 'y_file', 'out_name', 'best_estimator'], output_names=['curve_file'], function=learning_curve_fct), name='learning_curve_enet') wf.connect(aggregate_multimodal_metrics, 'X_file', learning_curve_enet, 'X_file') wf.connect(get_subjects_info, 'age_file', learning_curve_enet, 'y_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', learning_curve_enet, 'out_name') wf.connect(best_estimator_enet, 'best_estimator', learning_curve_enet, 'best_estimator') wf.connect(learning_curve_enet, 'curve_file', ds_pdf, 'learning_curve_enet') def plot_grid_scores_fct(gs_file, data_str): import pylab as plt import pickle, os import pandas as pd import numpy as np with open(gs_file) as fi: gs = pickle.load(fi) # build data frame names_params = gs[0].parameters.keys() c_names = names_params + ['m', 'sd'] df = pd.DataFrame([], columns=c_names) for g in gs: data_params = [g.parameters[n] for n in names_params] data_m = [np.abs(g.mean_validation_score), g.cv_validation_scores.std()] data = data_params + data_m df.loc[len(df)] = data # plot plt.figure(figsize=(5, 4 * len(names_params))) for i, p in enumerate(names_params): plt.subplot(len(names_params), 1, i + 1) x = df[p].values y = df['m'].values if type(x[0]) is str: rep_dict = {} for i, o in enumerate(np.unique(x)): rep_dict[o] = i x = np.array([rep_dict[o] for o in x]) x_ticks = rep_dict.keys() x_vals = map(rep_dict.get, x_ticks) plt.xticks(x_vals, x_ticks, rotation='vertical') u_x = x_vals u_y = [np.mean(df['m'][df[p] == v]) for v in rep_dict.keys()] else: u_x = np.unique(x) u_y = [np.mean(df['m'][df[p] == v]) for v in u_x] plt.scatter(x, y, color='black') plt.plot(u_x, u_y) plt.scatter(u_x, u_y, marker='+') plt.title(p) plt.tight_layout() fig_file = os.path.join(os.getcwd(), 'gs_plot_' + data_str + '.pdf') plt.savefig(fig_file) return fig_file plot_grid_scores = Node(util.Function(input_names=['gs_file', 'data_str'], output_names=['fig_file'], function=plot_grid_scores_fct), name='plot_grid_scores') wf.connect(prediction_gs_split, 'gs_file', plot_grid_scores, 'gs_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', plot_grid_scores, 'data_str') wf.connect(plot_grid_scores, 'fig_file', ds_pdf, 'gs_grid_score_plots') plot_random_grid_scores = Node(util.Function(input_names=['gs_file', 'data_str'], output_names=['fig_file'], function=plot_grid_scores_fct), name='plot_random_grid_scores') wf.connect(prediction_random_gs_split, 'gs_file', plot_random_grid_scores, 'gs_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', plot_random_grid_scores, 'data_str') wf.connect(plot_random_grid_scores, 'fig_file', ds_pdf, 'gs_random_grid_score_plots') def plot_validation_curve_fct(X_file, y_file, best_estimator, data_str): import pylab as plt import pickle, os import pandas as pd import numpy as np from sklearn.learning_curve import validation_curve from matplotlib.backends.backend_pdf import PdfPages X = np.load(X_file) y = np.load(y_file) params = { 'selection__percentile': [70, 75, 80, 85, 90, 95, 99, 100], 'regression_model__C': [1, 98, 106, 110, 130, 150, 170, 200, 1000, 10000, 14450], # [.0001, .001, .01, .1, 1, 10, 20, 30], 'regression_model__epsilon': [0, .005, .01, .05, .1, 1, 5, 10], 'regression_model__loss': ['epsilon_insensitive', 'squared_epsilon_insensitive'], # 'regression_model__nu': [0.01, .25, .5, .75, .8, .85, .9, .95, .99], } fig_file = os.path.abspath('validation_curve_' + data_str + '.pdf') fig_file_log = os.path.abspath('validation_curve_log' + data_str + '.pdf') with PdfPages(fig_file, 'w') as pdf: with PdfPages(fig_file_log, 'w') as pdf_log: for p in sorted(params.keys()): param_range = params[p] if type(param_range[0]) is not str: train_scores, test_scores = validation_curve(best_estimator, X, y, param_name=p, param_range=param_range, cv=5, n_jobs=5) train_scores_mean = np.mean(train_scores, axis=1) train_scores_std = np.std(train_scores, axis=1) test_scores_mean = np.mean(test_scores, axis=1) test_scores_std = np.std(test_scores, axis=1) # LINEAR AXIS plt.title('Validation Curve') plt.xlabel(p) plt.ylim(0.0, 1.1) plt.plot(param_range, train_scores_mean, label='Training score', color='r') plt.fill_between(param_range, train_scores_mean - train_scores_std, train_scores_mean + train_scores_std, alpha=0.2, color='r') plt.plot(param_range, test_scores_mean, label='Cross-validation score', color='g') plt.fill_between(param_range, test_scores_mean - test_scores_std, test_scores_mean + test_scores_std, alpha=0.2, color='g') plt.legend(loc='best') pdf.savefig() plt.close() # LOG AXIS plt.title('Validation Curve') plt.xlabel(p) plt.ylim(0.0, 1.1) plt.semilogx(param_range, train_scores_mean, label='Training score', color='r') plt.fill_between(param_range, train_scores_mean - train_scores_std, train_scores_mean + train_scores_std, alpha=0.2, color='r') plt.semilogx(param_range, test_scores_mean, label='Cross-validation score', color='g') plt.fill_between(param_range, test_scores_mean - test_scores_std, test_scores_mean + test_scores_std, alpha=0.2, color='g') plt.legend(loc='best') pdf_log.savefig() plt.close() return fig_file, fig_file_log plot_validation_curve = Node(util.Function(input_names=['X_file', 'y_file', 'best_estimator', 'data_str'], output_names=['fig_file', 'fig_file_log'], function=plot_validation_curve_fct), name='plot_validation_curve') wf.connect(aggregate_multimodal_metrics, 'X_file', plot_validation_curve, 'X_file') wf.connect(get_subjects_info, 'age_file', plot_validation_curve, 'y_file') wf.connect(prediction_gs_split, 'best_estimator', plot_validation_curve, 'best_estimator') wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', plot_validation_curve, 'data_str') wf.connect(plot_validation_curve, 'fig_file', ds_pdf, 'validation_curve.@lin') wf.connect(plot_validation_curve, 'fig_file_log', ds_pdf, 'validation_curve.@log') def run_pca_fct(X_file, data_str): from sklearn.decomposition import PCA import numpy as np import pylab as plt import os from sklearn.feature_selection import VarianceThreshold from sklearn.grid_search import GridSearchCV from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import Imputer X = np.load(X_file) # fixme pipe imp = Imputer() X = imp.fit_transform(X) # fixme? pipe # remove low variance features var_thr = VarianceThreshold() X = var_thr.fit_transform(X) # fixme pipe normalize normalize = MinMaxScaler() X = normalize.fit_transform(X) # fixme? # remove low variance features var_thr = VarianceThreshold() X = var_thr.fit_transform(X) pca = PCA() p = pca.fit_transform(X) explained_var = sum(pca.explained_variance_ratio_) plt.plot(range(1, len(pca.explained_variance_ratio_) + 1), np.cumsum(pca.explained_variance_ratio_)) plt.title('explained var: %s' % explained_var) plt.xlabel('n_components') plt.tight_layout() out_fig = os.path.join(os.getcwd(), 'pca_var_%s.pdf' % data_str) plt.savefig(out_fig) return out_fig run_pca = Node(util.Function(input_names=['X_file', 'data_str'], output_names=['out_fig'], function=run_pca_fct), name='run_pca') wf.connect(aggregate_multimodal_metrics, 'X_file', run_pca, 'X_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_out_name', run_pca, 'data_str') wf.connect(run_pca, 'out_fig', ds_pdf, 'pca') ##################################### # RUN WF ##################################### wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def _run_workflow(args): # build pipeline for each subject, individually # ~ 5 min 20 sec per subject # (roughly 320 seconds) import os import os.path as op import sys import nipype.interfaces.io as nio import nipype.pipeline.engine as pe import nipype.interfaces.utility as util import nipype.interfaces.fsl.maths as fsl import glob import time from time import strftime from nipype import config as nyconfig resource_pool, config, subject_info, run_name, site_name = args sub_id = str(subject_info[0]) qap_type = config['qap_type'] if subject_info[1]: session_id = subject_info[1] else: session_id = "session_0" if subject_info[2]: scan_id = subject_info[2] else: scan_id = "scan_0" # Read and apply general settings in config keep_outputs = config.get('write_all_outputs', False) output_dir = op.join(config["output_directory"], run_name, sub_id, session_id, scan_id) try: os.makedirs(output_dir) except: if not op.isdir(output_dir): err = "[!] Output directory unable to be created.\n" \ "Path: %s\n\n" % output_dir raise Exception(err) else: pass log_dir = output_dir # set up logging nyconfig.update_config( {'logging': {'log_directory': log_dir, 'log_to_file': True}}) logging.update_logging(nyconfig) # take date+time stamp for run identification purposes unique_pipeline_id = strftime("%Y%m%d%H%M%S") pipeline_start_stamp = strftime("%Y-%m-%d_%H:%M:%S") pipeline_start_time = time.time() logger.info("Pipeline start time: %s" % pipeline_start_stamp) logger.info("Contents of resource pool:\n" + str(resource_pool)) logger.info("Configuration settings:\n" + str(config)) # for QAP spreadsheet generation only config.update({"subject_id": sub_id, "session_id": session_id, "scan_id": scan_id, "run_name": run_name}) if site_name: config["site_name"] = site_name workflow = pe.Workflow(name=scan_id) workflow.base_dir = op.join(config["working_directory"], sub_id, session_id) # set up crash directory workflow.config['execution'] = \ {'crashdump_dir': config["output_directory"]} # update that resource pool with what's already in the output directory for resource in os.listdir(output_dir): if (op.isdir(op.join(output_dir, resource)) and resource not in resource_pool.keys()): resource_pool[resource] = glob.glob(op.join(output_dir, resource, "*"))[0] # resource pool check invalid_paths = [] for resource in resource_pool.keys(): if not op.isfile(resource_pool[resource]): invalid_paths.append((resource, resource_pool[resource])) if len(invalid_paths) > 0: err = "\n\n[!] The paths provided in the subject list to the " \ "following resources are not valid:\n" for path_tuple in invalid_paths: err = err + path_tuple[0] + ": " + path_tuple[1] + "\n" err = err + "\n\n" raise Exception(err) # start connecting the pipeline if 'qap_' + qap_type not in resource_pool.keys(): from qap import qap_workflows as qw wf_builder = getattr(qw, 'qap_' + qap_type + '_workflow') workflow, resource_pool = wf_builder(workflow, resource_pool, config) # set up the datasinks new_outputs = 0 out_list = ['qap_' + qap_type] if keep_outputs: out_list = resource_pool.keys() # Save reports to out_dir if necessary if config.get('write_report', False): out_list += ['qap_mosaic'] # The functional temporal also has an FD plot if 'functional_temporal' in qap_type: out_list += ['qap_fd'] for output in out_list: # we use a check for len()==2 here to select those items in the # resource pool which are tuples of (node, node_output), instead # of the items which are straight paths to files # resource pool items which are in the tuple format are the # outputs that have been created in this workflow because they # were not present in the subject list YML (the starting resource # pool) and had to be generated if len(resource_pool[output]) == 2: ds = pe.Node(nio.DataSink(), name='datasink_%s' % output) ds.inputs.base_directory = output_dir node, out_file = resource_pool[output] workflow.connect(node, out_file, ds, output) new_outputs += 1 rt = {'id': sub_id, 'session': session_id, 'scan': scan_id, 'status': 'started'} # run the pipeline (if there is anything to do) if new_outputs > 0: workflow.write_graph( dotfilename=op.join(output_dir, run_name + ".dot"), simple_form=False) nc_per_subject = config.get('num_cores_per_subject', 1) runargs = {'plugin': 'Linear', 'plugin_args': {}} if nc_per_subject > 1: runargs['plugin'] = 'MultiProc', runargs['plugin_args'] = {'n_procs': nc_per_subject} try: workflow.run(**runargs) rt['status'] = 'finished' except Exception as e: # TODO We should be more specific here ... rt.update({'status': 'failed', 'msg': e.msg}) # ... however this is run inside a pool.map: do not raise Execption else: rt['status'] = 'cached' logger.info("\nEverything is already done for subject %s." % sub_id) # Remove working directory when done if not keep_outputs: try: work_dir = op.join(workflow.base_dir, scan_id) if op.exists(work_dir): import shutil shutil.rmtree(work_dir) except: logger.warn("Couldn\'t remove the working directory!") pass pipeline_end_stamp = strftime("%Y-%m-%d_%H:%M:%S") pipeline_end_time = time.time() logger.info("Elapsed time (minutes) since last start: %s" % ((pipeline_end_time - pipeline_start_time) / 60)) logger.info("Pipeline end time: %s" % pipeline_end_stamp) return rt
def builder(subject_id, subId, project_dir, data_dir, output_dir, output_final_dir, output_interm_dir, layout, anat=None, funcs=None, fmaps=None, task_name='', session=None, apply_trim=False, apply_dist_corr=False, apply_smooth=False, apply_filter=False, mni_template='2mm', apply_n4=True, ants_threads=8, readable_crash_files=False, write_logs=True): """ Core function that returns a workflow. See wfmaker for more details. Args: subject_id: name of subject folder for final outputted sub-folder name subId: abbreviate name of subject for intermediate outputted sub-folder name project_dir: full path to root of project data_dir: full path to raw data files output_dir: upper level output dir (others will be nested within this) output_final_dir: final preprocessed sub-dir name output_interm_dir: intermediate preprcess sub-dir name layout: BIDS layout instance """ ################## ### PATH SETUP ### ################## if session is not None: session = int(session) if session < 10: session = '0' + str(session) else: session = str(session) # Set MNI template MNItemplate = os.path.join(get_resource_path(), 'MNI152_T1_' + mni_template + '_brain.nii.gz') MNImask = os.path.join(get_resource_path(), 'MNI152_T1_' + mni_template + '_brain_mask.nii.gz') MNItemplatehasskull = os.path.join(get_resource_path(), 'MNI152_T1_' + mni_template + '.nii.gz') # Set ANTs files bet_ants_template = os.path.join(get_resource_path(), 'OASIS_template.nii.gz') bet_ants_prob_mask = os.path.join( get_resource_path(), 'OASIS_BrainCerebellumProbabilityMask.nii.gz') bet_ants_registration_mask = os.path.join( get_resource_path(), 'OASIS_BrainCerebellumRegistrationMask.nii.gz') ################################# ### NIPYPE IMPORTS AND CONFIG ### ################################# # Update nipype global config because workflow.config[] = ..., doesn't seem to work # Can't store nipype config/rc file in container anyway so set them globaly before importing and setting up workflow as suggested here: http://nipype.readthedocs.io/en/latest/users/config_file.html#config-file # Create subject's intermediate directory before configuring nipype and the workflow because that's where we'll save log files in addition to intermediate files if not os.path.exists(os.path.join(output_interm_dir, subId, 'logs')): os.makedirs(os.path.join(output_interm_dir, subId, 'logs')) log_dir = os.path.join(output_interm_dir, subId, 'logs') from nipype import config if readable_crash_files: cfg = dict(execution={'crashfile_format': 'txt'}) config.update_config(cfg) config.update_config({ 'logging': { 'log_directory': log_dir, 'log_to_file': write_logs }, 'execution': { 'crashdump_dir': log_dir } }) from nipype import logging logging.update_logging(config) # Now import everything else from nipype.interfaces.io import DataSink from nipype.interfaces.utility import Merge, IdentityInterface from nipype.pipeline.engine import Node, Workflow from nipype.interfaces.nipy.preprocess import ComputeMask from nipype.algorithms.rapidart import ArtifactDetect from nipype.interfaces.ants.segmentation import BrainExtraction, N4BiasFieldCorrection from nipype.interfaces.ants import Registration, ApplyTransforms from nipype.interfaces.fsl import MCFLIRT, TOPUP, ApplyTOPUP from nipype.interfaces.fsl.maths import MeanImage from nipype.interfaces.fsl import Merge as MERGE from nipype.interfaces.fsl.utils import Smooth from nipype.interfaces.nipy.preprocess import Trim from .interfaces import Plot_Coregistration_Montage, Plot_Quality_Control, Plot_Realignment_Parameters, Create_Covariates, Down_Sample_Precision, Create_Encoding_File, Filter_In_Mask ################## ### INPUT NODE ### ################## # Turn functional file list into interable Node func_scans = Node(IdentityInterface(fields=['scan']), name='func_scans') func_scans.iterables = ('scan', funcs) # Get TR for use in filtering below; we're assuming all BOLD runs have the same TR tr_length = layout.get_metadata(funcs[0])['RepetitionTime'] ##################################### ## TRIM ## ##################################### if apply_trim: trim = Node(Trim(), name='trim') trim.inputs.begin_index = apply_trim ##################################### ## DISTORTION CORRECTION ## ##################################### if apply_dist_corr: # Get fmap file locations fmaps = [ f.filename for f in layout.get( subject=subId, modality='fmap', extensions='.nii.gz') ] if not fmaps: raise IOError( "Distortion Correction requested but field map scans not found..." ) # Get fmap metadata totalReadoutTimes, measurements, fmap_pes = [], [], [] for i, fmap in enumerate(fmaps): # Grab total readout time for each fmap totalReadoutTimes.append( layout.get_metadata(fmap)['TotalReadoutTime']) # Grab measurements (for some reason pyBIDS doesn't grab dcm_meta... fields from side-car json file and json.load, doesn't either; so instead just read the header using nibabel to determine number of scans) measurements.append(nib.load(fmap).header['dim'][4]) # Get phase encoding direction fmap_pe = layout.get_metadata(fmap)["PhaseEncodingDirection"] fmap_pes.append(fmap_pe) encoding_file_writer = Node(interface=Create_Encoding_File(), name='create_encoding') encoding_file_writer.inputs.totalReadoutTimes = totalReadoutTimes encoding_file_writer.inputs.fmaps = fmaps encoding_file_writer.inputs.fmap_pes = fmap_pes encoding_file_writer.inputs.measurements = measurements encoding_file_writer.inputs.file_name = 'encoding_file.txt' merge_to_file_list = Node(interface=Merge(2), infields=['in1', 'in2'], name='merge_to_file_list') merge_to_file_list.inputs.in1 = fmaps[0] merge_to_file_list.inputs.in1 = fmaps[1] # Merge AP and PA distortion correction scans merger = Node(interface=MERGE(dimension='t'), name='merger') merger.inputs.output_type = 'NIFTI_GZ' merger.inputs.in_files = fmaps merger.inputs.merged_file = 'merged_epi.nii.gz' # Create distortion correction map topup = Node(interface=TOPUP(), name='topup') topup.inputs.output_type = 'NIFTI_GZ' # Apply distortion correction to other scans apply_topup = Node(interface=ApplyTOPUP(), name='apply_topup') apply_topup.inputs.output_type = 'NIFTI_GZ' apply_topup.inputs.method = 'jac' apply_topup.inputs.interp = 'spline' ################################### ### REALIGN ### ################################### realign_fsl = Node(MCFLIRT(), name="realign") realign_fsl.inputs.cost = 'mutualinfo' realign_fsl.inputs.mean_vol = True realign_fsl.inputs.output_type = 'NIFTI_GZ' realign_fsl.inputs.save_mats = True realign_fsl.inputs.save_rms = True realign_fsl.inputs.save_plots = True ################################### ### MEAN EPIs ### ################################### # For coregistration after realignment mean_epi = Node(MeanImage(), name='mean_epi') mean_epi.inputs.dimension = 'T' # For after normalization is done to plot checks mean_norm_epi = Node(MeanImage(), name='mean_norm_epi') mean_norm_epi.inputs.dimension = 'T' ################################### ### MASK, ART, COV CREATION ### ################################### compute_mask = Node(ComputeMask(), name='compute_mask') compute_mask.inputs.m = .05 art = Node(ArtifactDetect(), name='art') art.inputs.use_differences = [True, False] art.inputs.use_norm = True art.inputs.norm_threshold = 1 art.inputs.zintensity_threshold = 3 art.inputs.mask_type = 'file' art.inputs.parameter_source = 'FSL' make_cov = Node(Create_Covariates(), name='make_cov') ################################ ### N4 BIAS FIELD CORRECTION ### ################################ if apply_n4: n4_correction = Node(N4BiasFieldCorrection(), name='n4_correction') n4_correction.inputs.copy_header = True n4_correction.inputs.save_bias = False n4_correction.inputs.num_threads = ants_threads n4_correction.inputs.input_image = anat ################################### ### BRAIN EXTRACTION ### ################################### brain_extraction_ants = Node(BrainExtraction(), name='brain_extraction') brain_extraction_ants.inputs.dimension = 3 brain_extraction_ants.inputs.use_floatingpoint_precision = 1 brain_extraction_ants.inputs.num_threads = ants_threads brain_extraction_ants.inputs.brain_probability_mask = bet_ants_prob_mask brain_extraction_ants.inputs.keep_temporary_files = 1 brain_extraction_ants.inputs.brain_template = bet_ants_template brain_extraction_ants.inputs.extraction_registration_mask = bet_ants_registration_mask brain_extraction_ants.inputs.out_prefix = 'bet' ################################### ### COREGISTRATION ### ################################### coregistration = Node(Registration(), name='coregistration') coregistration.inputs.float = False coregistration.inputs.output_transform_prefix = "meanEpi2highres" coregistration.inputs.transforms = ['Rigid'] coregistration.inputs.transform_parameters = [(0.1, ), (0.1, )] coregistration.inputs.number_of_iterations = [[1000, 500, 250, 100]] coregistration.inputs.dimension = 3 coregistration.inputs.num_threads = ants_threads coregistration.inputs.write_composite_transform = True coregistration.inputs.collapse_output_transforms = True coregistration.inputs.metric = ['MI'] coregistration.inputs.metric_weight = [1] coregistration.inputs.radius_or_number_of_bins = [32] coregistration.inputs.sampling_strategy = ['Regular'] coregistration.inputs.sampling_percentage = [0.25] coregistration.inputs.convergence_threshold = [1e-08] coregistration.inputs.convergence_window_size = [10] coregistration.inputs.smoothing_sigmas = [[3, 2, 1, 0]] coregistration.inputs.sigma_units = ['mm'] coregistration.inputs.shrink_factors = [[4, 3, 2, 1]] coregistration.inputs.use_estimate_learning_rate_once = [True] coregistration.inputs.use_histogram_matching = [False] coregistration.inputs.initial_moving_transform_com = True coregistration.inputs.output_warped_image = True coregistration.inputs.winsorize_lower_quantile = 0.01 coregistration.inputs.winsorize_upper_quantile = 0.99 ################################### ### NORMALIZATION ### ################################### # Settings Explanations # Only a few key settings are worth adjusting and most others relate to how ANTs optimizer starts or iterates and won't make a ton of difference # Brian Avants referred to these settings as the last "best tested" when he was aligning fMRI data: https://github.com/ANTsX/ANTsRCore/blob/master/R/antsRegistration.R#L275 # Things that matter the most: # smoothing_sigmas: # how much gaussian smoothing to apply when performing registration, probably want the upper limit of this to match the resolution that the data is collected at e.g. 3mm # Old settings [[3,2,1,0]]*3 # shrink_factors # The coarseness with which to do registration # Old settings [[8,4,2,1]] * 3 # >= 8 may result is some problems causing big chunks of cortex with little fine grain spatial structure to be moved to other parts of cortex # Other settings # transform_parameters: # how much regularization to do for fitting that transformation # for syn this pertains to both the gradient regularization term, and the flow, and elastic terms. Leave the syn settings alone as they seem to be the most well tested across published data sets # radius_or_number_of_bins # This is the bin size for MI metrics and 32 is probably adequate for most use cases. Increasing this might increase precision (e.g. to 64) but takes exponentially longer # use_histogram_matching # Use image intensity distribution to guide registration # Leave it on for within modality registration (e.g. T1 -> MNI), but off for between modality registration (e.g. EPI -> T1) # convergence_threshold # threshold for optimizer # convergence_window_size # how many samples should optimizer average to compute threshold? # sampling_strategy # what strategy should ANTs use to initialize the transform. Regular here refers to approximately random sampling around the center of the image mass normalization = Node(Registration(), name='normalization') normalization.inputs.float = False normalization.inputs.collapse_output_transforms = True normalization.inputs.convergence_threshold = [1e-06] normalization.inputs.convergence_window_size = [10] normalization.inputs.dimension = 3 normalization.inputs.fixed_image = MNItemplate normalization.inputs.initial_moving_transform_com = True normalization.inputs.metric = ['MI', 'MI', 'CC'] normalization.inputs.metric_weight = [1.0] * 3 normalization.inputs.number_of_iterations = [[1000, 500, 250, 100], [1000, 500, 250, 100], [100, 70, 50, 20]] normalization.inputs.num_threads = ants_threads normalization.inputs.output_transform_prefix = 'anat2template' normalization.inputs.output_inverse_warped_image = True normalization.inputs.output_warped_image = True normalization.inputs.radius_or_number_of_bins = [32, 32, 4] normalization.inputs.sampling_percentage = [0.25, 0.25, 1] normalization.inputs.sampling_strategy = ['Regular', 'Regular', 'None'] normalization.inputs.shrink_factors = [[8, 4, 2, 1]] * 3 normalization.inputs.sigma_units = ['vox'] * 3 normalization.inputs.smoothing_sigmas = [[3, 2, 1, 0]] * 3 normalization.inputs.transforms = ['Rigid', 'Affine', 'SyN'] normalization.inputs.transform_parameters = [(0.1, ), (0.1, ), (0.1, 3.0, 0.0)] normalization.inputs.use_histogram_matching = True normalization.inputs.winsorize_lower_quantile = 0.005 normalization.inputs.winsorize_upper_quantile = 0.995 normalization.inputs.write_composite_transform = True # NEW SETTINGS (need to be adjusted; specifically shink_factors and smoothing_sigmas need to be the same length) # normalization = Node(Registration(), name='normalization') # normalization.inputs.float = False # normalization.inputs.collapse_output_transforms = True # normalization.inputs.convergence_threshold = [1e-06, 1e-06, 1e-07] # normalization.inputs.convergence_window_size = [10] # normalization.inputs.dimension = 3 # normalization.inputs.fixed_image = MNItemplate # normalization.inputs.initial_moving_transform_com = True # normalization.inputs.metric = ['MI', 'MI', 'CC'] # normalization.inputs.metric_weight = [1.0]*3 # normalization.inputs.number_of_iterations = [[1000, 500, 250, 100], # [1000, 500, 250, 100], # [100, 70, 50, 20]] # normalization.inputs.num_threads = ants_threads # normalization.inputs.output_transform_prefix = 'anat2template' # normalization.inputs.output_inverse_warped_image = True # normalization.inputs.output_warped_image = True # normalization.inputs.radius_or_number_of_bins = [32, 32, 4] # normalization.inputs.sampling_percentage = [0.25, 0.25, 1] # normalization.inputs.sampling_strategy = ['Regular', # 'Regular', # 'None'] # normalization.inputs.shrink_factors = [[4, 3, 2, 1]]*3 # normalization.inputs.sigma_units = ['vox']*3 # normalization.inputs.smoothing_sigmas = [[2, 1], [2, 1], [3, 2, 1, 0]] # normalization.inputs.transforms = ['Rigid', 'Affine', 'SyN'] # normalization.inputs.transform_parameters = [(0.1,), # (0.1,), # (0.1, 3.0, 0.0)] # normalization.inputs.use_histogram_matching = True # normalization.inputs.winsorize_lower_quantile = 0.005 # normalization.inputs.winsorize_upper_quantile = 0.995 # normalization.inputs.write_composite_transform = True ################################### ### APPLY TRANSFORMS AND SMOOTH ### ################################### merge_transforms = Node(Merge(2), iterfield=['in2'], name='merge_transforms') # Used for epi -> mni, via (coreg + norm) apply_transforms = Node(ApplyTransforms(), iterfield=['input_image'], name='apply_transforms') apply_transforms.inputs.input_image_type = 3 apply_transforms.inputs.float = False apply_transforms.inputs.num_threads = 12 apply_transforms.inputs.environ = {} apply_transforms.inputs.interpolation = 'BSpline' apply_transforms.inputs.invert_transform_flags = [False, False] apply_transforms.inputs.reference_image = MNItemplate # Used for t1 segmented -> mni, via (norm) apply_transform_seg = Node(ApplyTransforms(), name='apply_transform_seg') apply_transform_seg.inputs.input_image_type = 3 apply_transform_seg.inputs.float = False apply_transform_seg.inputs.num_threads = 12 apply_transform_seg.inputs.environ = {} apply_transform_seg.inputs.interpolation = 'MultiLabel' apply_transform_seg.inputs.invert_transform_flags = [False] apply_transform_seg.inputs.reference_image = MNItemplate ################################### ### PLOTS ### ################################### plot_realign = Node(Plot_Realignment_Parameters(), name="plot_realign") plot_qa = Node(Plot_Quality_Control(), name="plot_qa") plot_normalization_check = Node(Plot_Coregistration_Montage(), name="plot_normalization_check") plot_normalization_check.inputs.canonical_img = MNItemplatehasskull ############################################ ### FILTER, SMOOTH, DOWNSAMPLE PRECISION ### ############################################ # Use cosanlab_preproc for down sampling down_samp = Node(Down_Sample_Precision(), name="down_samp") # Use FSL for smoothing if apply_smooth: smooth = Node(Smooth(), name='smooth') if isinstance(apply_smooth, list): smooth.iterables = ("fwhm", apply_smooth) elif isinstance(apply_smooth, int) or isinstance(apply_smooth, float): smooth.inputs.fwhm = apply_smooth else: raise ValueError("apply_smooth must be a list or int/float") # Use cosanlab_preproc for low-pass filtering if apply_filter: lp_filter = Node(Filter_In_Mask(), name='lp_filter') lp_filter.inputs.mask = MNImask lp_filter.inputs.sampling_rate = tr_length lp_filter.inputs.high_pass_cutoff = 0 if isinstance(apply_filter, list): lp_filter.iterables = ("low_pass_cutoff", apply_filter) elif isinstance(apply_filter, int) or isinstance(apply_filter, float): lp_filter.inputs.low_pass_cutoff = apply_filter else: raise ValueError("apply_filter must be a list or int/float") ################### ### OUTPUT NODE ### ################### # Collect all final outputs in the output dir and get rid of file name additions datasink = Node(DataSink(), name='datasink') if session: datasink.inputs.base_directory = os.path.join(output_final_dir, subject_id) datasink.inputs.container = 'ses-' + session else: datasink.inputs.base_directory = output_final_dir datasink.inputs.container = subject_id # Remove substitutions data_dir_parts = data_dir.split('/')[1:] if session: prefix = ['_scan_'] + data_dir_parts + [subject_id] + [ 'ses-' + session ] + ['func'] else: prefix = ['_scan_'] + data_dir_parts + [subject_id] + ['func'] func_scan_names = [os.path.split(elem)[-1] for elem in funcs] to_replace = [] for elem in func_scan_names: bold_name = elem.split(subject_id + '_')[-1] bold_name = bold_name.split('.nii.gz')[0] to_replace.append(('..'.join(prefix + [elem]), bold_name)) datasink.inputs.substitutions = to_replace ##################### ### INIT WORKFLOW ### ##################### # If we have sessions provide the full path to the subject's intermediate directory # and only rely on workflow init to create the session container *within* that directory # Otherwise just point to the intermediate directory and let the workflow init create the subject container within the intermediate directory if session: workflow = Workflow(name='ses_' + session) workflow.base_dir = os.path.join(output_interm_dir, subId) else: workflow = Workflow(name=subId) workflow.base_dir = output_interm_dir ############################ ######### PART (1a) ######### # func -> discorr -> trim -> realign # OR # func -> trim -> realign # OR # func -> discorr -> realign # OR # func -> realign ############################ if apply_dist_corr: workflow.connect([(encoding_file_writer, topup, [('encoding_file', 'encoding_file')]), (encoding_file_writer, apply_topup, [('encoding_file', 'encoding_file')]), (merger, topup, [('merged_file', 'in_file')]), (func_scans, apply_topup, [('scan', 'in_files')]), (topup, apply_topup, [('out_fieldcoef', 'in_topup_fieldcoef'), ('out_movpar', 'in_topup_movpar')])]) if apply_trim: # Dist Corr + Trim workflow.connect([(apply_topup, trim, [('out_corrected', 'in_file') ]), (trim, realign_fsl, [('out_file', 'in_file')])]) else: # Dist Corr + No Trim workflow.connect([(apply_topup, realign_fsl, [('out_corrected', 'in_file')])]) else: if apply_trim: # No Dist Corr + Trim workflow.connect([(func_scans, trim, [('scan', 'in_file')]), (trim, realign_fsl, [('out_file', 'in_file')])]) else: # No Dist Corr + No Trim workflow.connect([ (func_scans, realign_fsl, [('scan', 'in_file')]), ]) ############################ ######### PART (1n) ######### # anat -> N4 -> bet # OR # anat -> bet ############################ if apply_n4: workflow.connect([(n4_correction, brain_extraction_ants, [('output_image', 'anatomical_image')])]) else: brain_extraction_ants.inputs.anatomical_image = anat ########################################## ############### PART (2) ################# # realign -> coreg -> mni (via t1) # t1 -> mni # covariate creation # plot creation ########################################### workflow.connect([ (realign_fsl, plot_realign, [('par_file', 'realignment_parameters')]), (realign_fsl, plot_qa, [('out_file', 'dat_img')]), (realign_fsl, art, [('out_file', 'realigned_files'), ('par_file', 'realignment_parameters')]), (realign_fsl, mean_epi, [('out_file', 'in_file')]), (realign_fsl, make_cov, [('par_file', 'realignment_parameters')]), (mean_epi, compute_mask, [('out_file', 'mean_volume')]), (compute_mask, art, [('brain_mask', 'mask_file')]), (art, make_cov, [('outlier_files', 'spike_id')]), (art, plot_realign, [('outlier_files', 'outliers')]), (plot_qa, make_cov, [('fd_outliers', 'fd_outliers')]), (brain_extraction_ants, coregistration, [('BrainExtractionBrain', 'fixed_image')]), (mean_epi, coregistration, [('out_file', 'moving_image')]), (brain_extraction_ants, normalization, [('BrainExtractionBrain', 'moving_image')]), (coregistration, merge_transforms, [('composite_transform', 'in2')]), (normalization, merge_transforms, [('composite_transform', 'in1')]), (merge_transforms, apply_transforms, [('out', 'transforms')]), (realign_fsl, apply_transforms, [('out_file', 'input_image')]), (apply_transforms, mean_norm_epi, [('output_image', 'in_file')]), (normalization, apply_transform_seg, [('composite_transform', 'transforms')]), (brain_extraction_ants, apply_transform_seg, [('BrainExtractionSegmentation', 'input_image')]), (mean_norm_epi, plot_normalization_check, [('out_file', 'wra_img')]) ]) ################################################## ################### PART (3) ##################### # epi (in mni) -> filter -> smooth -> down sample # OR # epi (in mni) -> filter -> down sample # OR # epi (in mni) -> smooth -> down sample # OR # epi (in mni) -> down sample ################################################### if apply_filter: workflow.connect([(apply_transforms, lp_filter, [('output_image', 'in_file')])]) if apply_smooth: # Filtering + Smoothing workflow.connect([(lp_filter, smooth, [('out_file', 'in_file')]), (smooth, down_samp, [('smoothed_file', 'in_file') ])]) else: # Filtering + No Smoothing workflow.connect([(lp_filter, down_samp, [('out_file', 'in_file')]) ]) else: if apply_smooth: # No Filtering + Smoothing workflow.connect([ (apply_transforms, smooth, [('output_image', 'in_file')]), (smooth, down_samp, [('smoothed_file', 'in_file')]) ]) else: # No Filtering + No Smoothing workflow.connect([(apply_transforms, down_samp, [('output_image', 'in_file')])]) ########################################## ############### PART (4) ################# # down sample -> save # plots -> save # covs -> save # t1 (in mni) -> save # t1 segmented masks (in mni) -> save # realignment parms -> save ########################################## workflow.connect([ (down_samp, datasink, [('out_file', 'functional.@down_samp')]), (plot_realign, datasink, [('plot', 'functional.@plot_realign')]), (plot_qa, datasink, [('plot', 'functional.@plot_qa')]), (plot_normalization_check, datasink, [('plot', 'functional.@plot_normalization')]), (make_cov, datasink, [('covariates', 'functional.@covariates')]), (normalization, datasink, [('warped_image', 'structural.@normanat')]), (apply_transform_seg, datasink, [('output_image', 'structural.@normanatseg')]), (realign_fsl, datasink, [('par_file', 'functional.@motionparams')]) ]) if not os.path.exists(os.path.join(output_dir, 'pipeline.png')): workflow.write_graph(dotfilename=os.path.join(output_dir, 'pipeline'), format='png') print(f"Creating workflow for subject: {subject_id}") if ants_threads != 8: print( f"ANTs will utilize the user-requested {ants_threads} threads for parallel processing." ) return workflow
def learning_predict_data_2samp_wf(working_dir, ds_dir, in_data_name_list, subjects_selection_crit_dict, subjects_selection_crit_names_list, aggregated_subjects_dir, target_list, use_n_procs, plugin_name, confound_regression=[False, True], run_cv=False, n_jobs_cv=1, run_tuning=False, run_2sample_training=False, aggregated_subjects_dir_nki=None, subjects_selection_crit_dict_nki=None, subjects_selection_crit_name_nki=None, reverse_split=False, random_state_nki=666, run_learning_curve=False, life_test_size=0.5): import os from nipype import config from nipype.pipeline.engine import Node, Workflow import nipype.interfaces.utility as util import nipype.interfaces.io as nio from itertools import chain from learning_utils import aggregate_multimodal_metrics_fct, run_prediction_split_fct, \ backproject_and_split_weights_fct, select_subjects_fct, select_multimodal_X_fct, learning_curve_plot import pandas as pd ############################################################################################################### # GENERAL SETTINGS wf = Workflow(name='learning_predict_data_2samp_wf') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': False, 'remove_unnecessary_outputs': False, 'job_finished_timeout': 120}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(working_dir, 'crash') ds = Node(nio.DataSink(), name='ds') ds.inputs.base_directory = os.path.join(ds_dir, 'group_learning_prepare_data') ds.inputs.regexp_substitutions = [ # ('subject_id_', ''), ('_parcellation_', ''), ('_bp_freqs_', 'bp_'), ('_extraction_method_', ''), ('_subject_id_[A0-9]*/', '') ] ds_pdf = Node(nio.DataSink(), name='ds_pdf') ds_pdf.inputs.base_directory = os.path.join(ds_dir, 'pdfs') ds_pdf.inputs.parameterization = False ############################################################################################################### # ensure in_data_name_list is list of lists in_data_name_list = [i if type(i) == list else [i] for i in in_data_name_list] in_data_name_list_unique = list(set(chain.from_iterable(in_data_name_list))) ############################################################################################################### # SET ITERATORS in_data_name_infosource = Node(util.IdentityInterface(fields=['in_data_name']), name='in_data_name_infosource') in_data_name_infosource.iterables = ('in_data_name', in_data_name_list_unique) multimodal_in_data_name_infosource = Node(util.IdentityInterface(fields=['multimodal_in_data_name']), name='multimodal_in_data_name_infosource') multimodal_in_data_name_infosource.iterables = ('multimodal_in_data_name', in_data_name_list) subject_selection_infosource = Node(util.IdentityInterface(fields=['selection_criterium']), name='subject_selection_infosource') subject_selection_infosource.iterables = ('selection_criterium', subjects_selection_crit_names_list) target_infosource = Node(util.IdentityInterface(fields=['target_name']), name='target_infosource') target_infosource.iterables = ('target_name', target_list) ############################################################################################################### # COMPILE LIFE DATA ############################################################################################################### ############################################################################################################### # GET INFO AND SELECT FILES df_all_subjects_pickle_file = os.path.join(aggregated_subjects_dir, 'df_all_subjects_pickle_file/df_all.pkl') df = pd.read_pickle(df_all_subjects_pickle_file) # build lookup dict for unimodal data X_file_template = 'X_file/_in_data_name_{in_data_name}/vectorized_aggregated_data.npy' info_file_template = 'unimodal_backprojection_info_file/_in_data_name_{in_data_name}/unimodal_backprojection_info.pkl' unimodal_lookup_dict = {} for k in in_data_name_list_unique: unimodal_lookup_dict[k] = {'X_file': os.path.join(aggregated_subjects_dir, X_file_template.format( in_data_name=k)), 'unimodal_backprojection_info_file': os.path.join(aggregated_subjects_dir, info_file_template.format( in_data_name=k)) } ############################################################################################################### # AGGREGATE MULTIMODAL METRICS # stack single modality arrays horizontally aggregate_multimodal_metrics = Node(util.Function(input_names=['multimodal_list', 'unimodal_lookup_dict'], output_names=['X_multimodal_file', 'multimodal_backprojection_info', 'multimodal_name'], function=aggregate_multimodal_metrics_fct), name='aggregate_multimodal_metrics') wf.connect(multimodal_in_data_name_infosource, 'multimodal_in_data_name', aggregate_multimodal_metrics, 'multimodal_list') aggregate_multimodal_metrics.inputs.unimodal_lookup_dict = unimodal_lookup_dict ############################################################################################################### # GET INDEXER FOR SUBJECTS OF INTEREST (as defined by selection criterium) select_subjects = Node(util.Function(input_names=['df_all_subjects_pickle_file', 'subjects_selection_crit_dict', 'selection_criterium'], output_names=['df_use_file', 'df_use_pickle_file', 'subjects_selection_index'], function=select_subjects_fct), name='select_subjects') select_subjects.inputs.df_all_subjects_pickle_file = df_all_subjects_pickle_file select_subjects.inputs.subjects_selection_crit_dict = subjects_selection_crit_dict wf.connect(subject_selection_infosource, 'selection_criterium', select_subjects, 'selection_criterium') ############################################################################################################### # SELECT MULITMODAL X # select subjects (rows) from multimodal X according indexer select_multimodal_X = Node(util.Function(input_names=['X_multimodal_file', 'subjects_selection_index', 'selection_criterium'], output_names=['X_multimodal_selected_file'], function=select_multimodal_X_fct), name='select_multimodal_X') wf.connect(aggregate_multimodal_metrics, 'X_multimodal_file', select_multimodal_X, 'X_multimodal_file') wf.connect(select_subjects, 'subjects_selection_index', select_multimodal_X, 'subjects_selection_index') ############################################################################################################### # COMPILE NKI DATA ############################################################################################################### if run_2sample_training: ############################################################################################################### # GET INFO AND SELECT FILES df_all_subjects_pickle_file_nki = os.path.join(aggregated_subjects_dir_nki, 'df_all_subjects_pickle_file/df_all.pkl') df_nki = pd.read_pickle(df_all_subjects_pickle_file_nki) # build lookup dict for unimodal data X_file_template = 'X_file/_in_data_name_{in_data_name}/vectorized_aggregated_data.npy' info_file_template = 'unimodal_backprojection_info_file/_in_data_name_{in_data_name}/unimodal_backprojection_info.pkl' unimodal_lookup_dict_nki = {} for k in in_data_name_list_unique: unimodal_lookup_dict_nki[k] = {'X_file': os.path.join(aggregated_subjects_dir_nki, X_file_template.format( in_data_name=k)), 'unimodal_backprojection_info_file': os.path.join( aggregated_subjects_dir_nki, info_file_template.format( in_data_name=k)) } ############################################################################################################### # AGGREGATE MULTIMODAL METRICS # stack single modality arrays horizontally aggregate_multimodal_metrics_nki = Node(util.Function(input_names=['multimodal_list', 'unimodal_lookup_dict'], output_names=['X_multimodal_file', 'multimodal_backprojection_info', 'multimodal_name'], function=aggregate_multimodal_metrics_fct), name='aggregate_multimodal_metrics_nki') wf.connect(multimodal_in_data_name_infosource, 'multimodal_in_data_name', aggregate_multimodal_metrics_nki, 'multimodal_list') aggregate_multimodal_metrics_nki.inputs.unimodal_lookup_dict = unimodal_lookup_dict_nki ############################################################################################################### # GET INDEXER FOR SUBJECTS OF INTEREST (as defined by selection criterium) select_subjects_nki = Node(util.Function(input_names=['df_all_subjects_pickle_file', 'subjects_selection_crit_dict', 'selection_criterium'], output_names=['df_use_file', 'df_use_pickle_file', 'subjects_selection_index'], function=select_subjects_fct), name='select_subjects_nki') select_subjects_nki.inputs.df_all_subjects_pickle_file = df_all_subjects_pickle_file_nki select_subjects_nki.inputs.subjects_selection_crit_dict = subjects_selection_crit_dict_nki select_subjects_nki.inputs.selection_criterium = subjects_selection_crit_name_nki ############################################################################################################### # SELECT MULITMODAL X # select subjects (rows) from multimodal X according indexer select_multimodal_X_nki = Node(util.Function(input_names=['X_multimodal_file', 'subjects_selection_index', 'selection_criterium'], output_names=['X_multimodal_selected_file'], function=select_multimodal_X_fct), name='select_multimodal_X_nki') wf.connect(aggregate_multimodal_metrics_nki, 'X_multimodal_file', select_multimodal_X_nki, 'X_multimodal_file') wf.connect(select_subjects_nki, 'subjects_selection_index', select_multimodal_X_nki, 'subjects_selection_index') ############################################################################################################### # RUN PREDICTION # prediction_node_dict = {} backprojection_node_dict = {} prediction_split = Node(util.Function(input_names=['X_file', 'target_name', 'selection_criterium', 'df_file', 'data_str', 'regress_confounds', 'run_cv', 'n_jobs_cv', 'run_tuning', 'X_file_nki', 'df_file_nki', 'reverse_split', 'random_state_nki', 'run_learning_curve', 'life_test_size'], output_names=['scatter_file', 'brain_age_scatter_file', 'df_life_out_file', 'df_nki_out_file', 'df_big_out_file', 'model_out_file', 'df_res_out_file', 'tuning_curve_file', 'scatter_file_cv', 'learning_curve_plot_file', 'learning_curve_df_file'], function=run_prediction_split_fct), name='prediction_split') backproject_and_split_weights = Node(util.Function(input_names=['trained_model_file', 'multimodal_backprojection_info', 'data_str', 'target_name'], output_names=['out_file_list', 'out_file_render_list'], function=backproject_and_split_weights_fct), name='backproject_and_split_weights') i = 0 for reg in confound_regression: the_out_node_str = 'single_source_model_reg_%s_' % (reg) prediction_node_dict[i] = prediction_split.clone(the_out_node_str) the_in_node = prediction_node_dict[i] the_in_node.inputs.regress_confounds = reg the_in_node.inputs.run_cv = run_cv the_in_node.inputs.n_jobs_cv = n_jobs_cv the_in_node.inputs.run_tuning = run_tuning the_in_node.inputs.reverse_split = reverse_split the_in_node.inputs.random_state_nki = random_state_nki the_in_node.inputs.run_learning_curve = run_learning_curve the_in_node.inputs.life_test_size = life_test_size wf.connect(select_multimodal_X, 'X_multimodal_selected_file', the_in_node, 'X_file') wf.connect(target_infosource, 'target_name', the_in_node, 'target_name') wf.connect(subject_selection_infosource, 'selection_criterium', the_in_node, 'selection_criterium') wf.connect(select_subjects, 'df_use_pickle_file', the_in_node, 'df_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_name', the_in_node, 'data_str') wf.connect(the_in_node, 'model_out_file', ds, the_out_node_str + 'trained_model') wf.connect(the_in_node, 'scatter_file', ds_pdf, the_out_node_str + 'scatter') wf.connect(the_in_node, 'brain_age_scatter_file', ds_pdf, the_out_node_str + 'brain_age_scatter') wf.connect(the_in_node, 'df_life_out_file', ds_pdf, the_out_node_str + 'predicted_life') wf.connect(the_in_node, 'df_nki_out_file', ds_pdf, the_out_node_str + 'predicted_nki') wf.connect(the_in_node, 'df_big_out_file', ds_pdf, the_out_node_str + 'predicted') wf.connect(the_in_node, 'df_res_out_file', ds_pdf, the_out_node_str + 'results_error') wf.connect(the_in_node, 'tuning_curve_file', ds_pdf, the_out_node_str + 'tuning_curve') wf.connect(the_in_node, 'scatter_file_cv', ds_pdf, the_out_node_str + 'scatter_cv') wf.connect(the_in_node, 'learning_curve_plot_file', ds_pdf, the_out_node_str + 'learning_curve_plot_file.@plot') wf.connect(the_in_node, 'learning_curve_df_file', ds_pdf, the_out_node_str + 'learning_curve_df_file.@df') # NKI if run_2sample_training: wf.connect(select_multimodal_X_nki, 'X_multimodal_selected_file', the_in_node, 'X_file_nki') wf.connect(select_subjects_nki, 'df_use_pickle_file', the_in_node, 'df_file_nki') else: the_in_node.inputs.df_file_nki = None the_in_node.inputs.X_file_nki = None # BACKPROJECT PREDICTION WEIGHTS # map weights back to single modality original format (e.g., nifti or matrix) the_out_node_str = 'backprojection_single_source_model_reg_%s_' % (reg) backprojection_node_dict[i] = backproject_and_split_weights.clone(the_out_node_str) the_from_node = prediction_node_dict[i] the_in_node = backprojection_node_dict[i] wf.connect(the_from_node, 'model_out_file', the_in_node, 'trained_model_file') wf.connect(aggregate_multimodal_metrics, 'multimodal_backprojection_info', the_in_node, 'multimodal_backprojection_info') wf.connect(aggregate_multimodal_metrics, 'multimodal_name', the_in_node, 'data_str') wf.connect(target_infosource, 'target_name', the_in_node, 'target_name') wf.connect(the_in_node, 'out_file_list', ds_pdf, the_out_node_str + '.@weights') wf.connect(the_in_node, 'out_file_render_list', ds_pdf, the_out_node_str + 'renders.@renders') i += 1 ############################################################################################################### # # RUN WF wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def process(self): # Enable the use of the the W3C PROV data model to capture and represent provenance in Nipype # config.enable_provenance() # Process time self.now = datetime.datetime.now().strftime("%Y%m%d_%H%M") if '_' in self.subject: self.subject = self.subject.split('_')[0] # old_subject = self.subject if self.global_conf.subject_session == '': cmp_deriv_subject_directory = os.path.join(self.output_directory, "cmp", self.subject) nipype_deriv_subject_directory = os.path.join( self.output_directory, "nipype", self.subject) else: cmp_deriv_subject_directory = os.path.join( self.output_directory, "cmp", self.subject, self.global_conf.subject_session) nipype_deriv_subject_directory = os.path.join( self.output_directory, "nipype", self.subject, self.global_conf.subject_session) self.subject = "_".join( (self.subject, self.global_conf.subject_session)) if not os.path.exists( os.path.join(nipype_deriv_subject_directory, "fMRI_pipeline")): try: os.makedirs( os.path.join(nipype_deriv_subject_directory, "fMRI_pipeline")) except os.error: print("%s was already existing" % os.path.join( nipype_deriv_subject_directory, "fMRI_pipeline")) # Initialization if os.path.isfile( os.path.join(nipype_deriv_subject_directory, "fMRI_pipeline", "pypeline.log")): os.unlink( os.path.join(nipype_deriv_subject_directory, "fMRI_pipeline", "pypeline.log")) config.update_config({ 'logging': { 'log_directory': os.path.join(nipype_deriv_subject_directory, "fMRI_pipeline"), 'log_to_file': True }, 'execution': { 'remove_unnecessary_outputs': False, 'stop_on_first_crash': True, 'stop_on_first_rerun': False, 'use_relative_paths': True, 'crashfile_format': "txt" } }) logging.update_logging(config) iflogger = logging.getLogger('nipype.interface') iflogger.info("**** Processing ****") flow = self.create_pipeline_flow( cmp_deriv_subject_directory=cmp_deriv_subject_directory, nipype_deriv_subject_directory=nipype_deriv_subject_directory) flow.write_graph(graph2use='colored', format='svg', simple_form=False) # try: if (self.number_of_cores != 1): flow.run(plugin='MultiProc', plugin_args={'n_procs': self.number_of_cores}) else: flow.run() # self.fill_stages_outputs() iflogger.info("**** Processing finished ****") return True, 'Processing successful'
if __name__ == '__main__': import sys import datetime, time from nipype import config from dmri_pipe1_prepro import do_pipe1_prepro from dmri_pipe2_tractscript import script_tracking, trackwait from dmri_pipe3_projection import do_pipe3_projection from dmri_pipe4_distmat import do_pipe4_distmat from dmri_pipe5_distmat_lr import do_pipe5_distmat_lr from dmri_pipe_cleanup import do_cleanup, do_wrapup cfg = dict(logging=dict(workflow_level = 'INFO'), execution={'remove_unnecessary_outputs': False, 'job_finished_timeout': 120, 'stop_on_first_rerun': False, 'stop_on_first_crash': True} ) config.update_config(cfg) tract_number = 5000 tract_step = 0.3 freesurfer_dir = '/scr/kalifornien1/data/nki_enhanced/freesurfer' data_dir = '/scr/kalifornien1/data/nki_enhanced/dicoms/diff_2_nii' # freesurfer_dir = '/scr/kongo2/NKI_ENH/freesurfer' # data_dir = '/scr/kongo2/NKI_ENH/dMRI' data_template = "%s/DTI_mx_137/%s" is_LH = True is_RH = False pipe_dict = dict([('0',0),('1',0),('2',1),('21',1),('22',2),('3',3),('4',4),('41',4),('42',5),('5',6)]) pipe_stop = 7
def segmentation(projectid, subjectid, sessionid, master_config, onlyT1=True, pipeline_name=''): import os.path import nipype.pipeline.engine as pe import nipype.interfaces.io as nio from nipype.interfaces import ants from nipype.interfaces.utility import IdentityInterface, Function, Merge # Set universal pipeline options from nipype import config config.update_config(master_config) assert config.get('execution', 'plugin') == master_config['execution']['plugin'] from PipeLineFunctionHelpers import ClipT1ImageWithBrainMask from WorkupT1T2BRAINSCut import CreateBRAINSCutWorkflow from utilities.distributed import modify_qsub_args from SEMTools import BRAINSSnapShotWriter baw200 = pe.Workflow(name=pipeline_name) # HACK: print for debugging for key, itme in master_config.items(): print "-" * 30 print key, ":", itme print "-" * 30 #END HACK inputsSpec = pe.Node(interface=IdentityInterface(fields=['t1_average', 't2_average', 'template_t1', 'hncma-atlas', 'LMIatlasToSubject_tx', 'inputLabels', 'inputHeadLabels', 'posteriorImages', 'TissueClassifyatlasToSubjectInverseTransform', 'UpdatedPosteriorsList']), run_without_submitting=True, name='inputspec') # outputsSpec = pe.Node(interface=IdentityInterface(fields=[...]), # run_without_submitting=True, name='outputspec') currentClipT1ImageWithBrainMaskName = 'ClipT1ImageWithBrainMask_' + str(subjectid) + "_" + str(sessionid) ClipT1ImageWithBrainMaskNode = pe.Node(interface=Function(function=ClipT1ImageWithBrainMask, input_names=['t1_image', 'brain_labels', 'clipped_file_name'], output_names=['clipped_file']), name=currentClipT1ImageWithBrainMaskName) ClipT1ImageWithBrainMaskNode.inputs.clipped_file_name = 'clipped_from_BABC_labels_t1.nii.gz' baw200.connect([(inputsSpec, ClipT1ImageWithBrainMaskNode, [('t1_average', 't1_image'), ('inputLabels', 'brain_labels')])]) currentAtlasToSubjectantsRegistration = 'AtlasToSubjectANTsRegistration_' + str(subjectid) + "_" + str(sessionid) AtlasToSubjectantsRegistration = pe.Node(interface=ants.Registration(), name=currentAtlasToSubjectantsRegistration) AtlasToSubjectantsRegistration.inputs.dimension = 3 AtlasToSubjectantsRegistration.inputs.transforms = ["Affine", "SyN"] AtlasToSubjectantsRegistration.inputs.transform_parameters = [[0.1], [0.15, 3.0, 0.0]] AtlasToSubjectantsRegistration.inputs.metric = ['Mattes', 'CC'] AtlasToSubjectantsRegistration.inputs.sampling_strategy = ['Regular', None] AtlasToSubjectantsRegistration.inputs.sampling_percentage = [1.0, 1.0] AtlasToSubjectantsRegistration.inputs.metric_weight = [1.0, 1.0] AtlasToSubjectantsRegistration.inputs.radius_or_number_of_bins = [32, 4] AtlasToSubjectantsRegistration.inputs.number_of_iterations = [[1000, 1000, 1000], [10000, 500, 500, 200]] AtlasToSubjectantsRegistration.inputs.convergence_threshold = [5e-7, 5e-7] AtlasToSubjectantsRegistration.inputs.convergence_window_size = [25, 25] AtlasToSubjectantsRegistration.inputs.use_histogram_matching = [True, True] AtlasToSubjectantsRegistration.inputs.shrink_factors = [[4, 2, 1], [5, 4, 2, 1]] AtlasToSubjectantsRegistration.inputs.smoothing_sigmas = [[4, 2, 0], [5, 4, 2, 0]] AtlasToSubjectantsRegistration.inputs.sigma_units = ["vox","vox"] AtlasToSubjectantsRegistration.inputs.use_estimate_learning_rate_once = [False, False] AtlasToSubjectantsRegistration.inputs.write_composite_transform = True AtlasToSubjectantsRegistration.inputs.collapse_output_transforms = True AtlasToSubjectantsRegistration.inputs.output_transform_prefix = 'AtlasToSubject_' AtlasToSubjectantsRegistration.inputs.winsorize_lower_quantile = 0.025 AtlasToSubjectantsRegistration.inputs.winsorize_upper_quantile = 0.975 AtlasToSubjectantsRegistration.inputs.collapse_linear_transforms_to_fixed_image_header = False AtlasToSubjectantsRegistration.inputs.output_warped_image = 'atlas2subject.nii.gz' AtlasToSubjectantsRegistration.inputs.output_inverse_warped_image = 'subject2atlas.nii.gz' baw200.connect([(inputsSpec, AtlasToSubjectantsRegistration, [('LMIatlasToSubject_tx', 'initial_moving_transform'), ('t1_average', 'fixed_image'), ('template_t1', 'moving_image')]) ]) myLocalSegWF = CreateBRAINSCutWorkflow(projectid, subjectid, sessionid, master_config['queue'], master_config['long_q'], t1Only=onlyT1) MergeStage2AverageImagesName = "99_mergeAvergeStage2Images_" + str(sessionid) MergeStage2AverageImages = pe.Node(interface=Merge(2), run_without_submitting=True, name=MergeStage2AverageImagesName) baw200.connect([(inputsSpec, myLocalSegWF, [('t1_average', 'inputspec.T1Volume'), ('posteriorImages', "inputspec.posteriorDictionary"), ('inputLabels', 'inputspec.RegistrationROI'),]), (inputsSpec, MergeStage2AverageImages, [('t1_average', 'in1')]), (AtlasToSubjectantsRegistration, myLocalSegWF, [('composite_transform', 'inputspec.atlasToSubjectTransform')]) ]) if not onlyT1: baw200.connect([(inputsSpec, myLocalSegWF, [('t2_average', 'inputspec.T2Volume')]), (inputsSpec, MergeStage2AverageImages, [('t2_average', 'in2')])]) file_count = 15 # Count of files to merge into MergeSessionSubjectToAtlas else: file_count = 14 # Count of files to merge into MergeSessionSubjectToAtlas ## NOTE: Element 0 of AccumulatePriorsList is the accumulated GM tissue # baw200.connect([(AccumulateLikeTissuePosteriorsNode, myLocalSegWF, # [(('AccumulatePriorsList', getListIndex, 0), "inputspec.TotalGM")]), # ]) ### Now define where the final organized outputs should go. DataSink = pe.Node(nio.DataSink(), name="CleanedDenoisedSegmentation_DS_" + str(subjectid) + "_" + str(sessionid)) DataSink.overwrite = master_config['ds_overwrite'] DataSink.inputs.base_directory = master_config['resultdir'] # DataSink.inputs.regexp_substitutions = GenerateOutputPattern(projectid, subjectid, sessionid,'BRAINSCut') # DataSink.inputs.regexp_substitutions = GenerateBRAINSCutImagesOutputPattern(projectid, subjectid, sessionid) DataSink.inputs.substitutions = [('Segmentations', os.path.join(projectid, subjectid, sessionid, 'CleanedDenoisedRFSegmentations')), ('subjectANNLabel_', ''), ('ANNContinuousPrediction', ''), ('subject.nii.gz', '.nii.gz'), ('_seg.nii.gz', '_seg.nii.gz'), ('.nii.gz', '_seg.nii.gz'), ('_seg_seg', '_seg')] baw200.connect([(myLocalSegWF, DataSink, [('outputspec.outputBinaryLeftCaudate', 'Segmentations.@LeftCaudate'), ('outputspec.outputBinaryRightCaudate', 'Segmentations.@RightCaudate'), ('outputspec.outputBinaryLeftHippocampus', 'Segmentations.@LeftHippocampus'), ('outputspec.outputBinaryRightHippocampus', 'Segmentations.@RightHippocampus'), ('outputspec.outputBinaryLeftPutamen', 'Segmentations.@LeftPutamen'), ('outputspec.outputBinaryRightPutamen', 'Segmentations.@RightPutamen'), ('outputspec.outputBinaryLeftThalamus', 'Segmentations.@LeftThalamus'), ('outputspec.outputBinaryRightThalamus', 'Segmentations.@RightThalamus'), ('outputspec.outputBinaryLeftAccumben', 'Segmentations.@LeftAccumben'), ('outputspec.outputBinaryRightAccumben', 'Segmentations.@RightAccumben'), ('outputspec.outputBinaryLeftGlobus', 'Segmentations.@LeftGlobus'), ('outputspec.outputBinaryRightGlobus', 'Segmentations.@RightGlobus'), ('outputspec.outputLabelImageName', 'Segmentations.@LabelImageName'), ('outputspec.outputCSVFileName', 'Segmentations.@CSVFileName')]), # (myLocalSegWF, DataSink, [('outputspec.cleaned_labels', 'Segmentations.@cleaned_labels')]) ]) MergeStage2BinaryVolumesName = "99_MergeStage2BinaryVolumes_" + str(sessionid) MergeStage2BinaryVolumes = pe.Node(interface=Merge(12), run_without_submitting=True, name=MergeStage2BinaryVolumesName) baw200.connect([(myLocalSegWF, MergeStage2BinaryVolumes, [('outputspec.outputBinaryLeftAccumben', 'in1'), ('outputspec.outputBinaryLeftCaudate', 'in2'), ('outputspec.outputBinaryLeftPutamen', 'in3'), ('outputspec.outputBinaryLeftGlobus', 'in4'), ('outputspec.outputBinaryLeftThalamus', 'in5'), ('outputspec.outputBinaryLeftHippocampus', 'in6'), ('outputspec.outputBinaryRightAccumben', 'in7'), ('outputspec.outputBinaryRightCaudate', 'in8'), ('outputspec.outputBinaryRightPutamen', 'in9'), ('outputspec.outputBinaryRightGlobus', 'in10'), ('outputspec.outputBinaryRightThalamus', 'in11'), ('outputspec.outputBinaryRightHippocampus', 'in12')]) ]) ## SnapShotWriter for Segmented result checking: SnapShotWriterNodeName = "SnapShotWriter_" + str(sessionid) SnapShotWriter = pe.Node(interface=BRAINSSnapShotWriter(), name=SnapShotWriterNodeName) SnapShotWriter.inputs.outputFilename = 'snapShot' + str(sessionid) + '.png' # output specification SnapShotWriter.inputs.inputPlaneDirection = [2, 1, 1, 1, 1, 0, 0] SnapShotWriter.inputs.inputSliceToExtractInPhysicalPoint = [-3, -7, -3, 5, 7, 22, -22] baw200.connect([(MergeStage2AverageImages, SnapShotWriter, [('out', 'inputVolumes')]), (MergeStage2BinaryVolumes, SnapShotWriter, [('out', 'inputBinaryVolumes')]), (SnapShotWriter, DataSink, [('outputFilename', 'Segmentations.@outputSnapShot')]) ]) currentAntsLabelWarpToSubject = 'AntsLabelWarpToSubject' + str(subjectid) + "_" + str(sessionid) AntsLabelWarpToSubject = pe.Node(interface=ants.ApplyTransforms(), name=currentAntsLabelWarpToSubject) AntsLabelWarpToSubject.inputs.dimension = 3 AntsLabelWarpToSubject.inputs.output_image = 'warped_hncma_atlas_seg.nii.gz' AntsLabelWarpToSubject.inputs.interpolation = "MultiLabel" baw200.connect([(AtlasToSubjectantsRegistration, AntsLabelWarpToSubject, [('composite_transform', 'transforms')]), (inputsSpec, AntsLabelWarpToSubject, [('t1_average', 'reference_image'), ('hncma-atlas', 'input_image')]) ]) ##### ### Now define where the final organized outputs should go. AntsLabelWarpedToSubject_DSName = "AntsLabelWarpedToSubject_DS_" + str(sessionid) AntsLabelWarpedToSubject_DS = pe.Node(nio.DataSink(), name=AntsLabelWarpedToSubject_DSName) AntsLabelWarpedToSubject_DS.overwrite = master_config['ds_overwrite'] AntsLabelWarpedToSubject_DS.inputs.base_directory = master_config['resultdir'] AntsLabelWarpedToSubject_DS.inputs.substitutions = [('AntsLabelWarpedToSubject', os.path.join(projectid, subjectid, sessionid, 'AntsLabelWarpedToSubject'))] baw200.connect([(AntsLabelWarpToSubject, AntsLabelWarpedToSubject_DS, [('output_image', 'AntsLabelWarpedToSubject')])]) MergeSessionSubjectToAtlasName = "99_MergeSessionSubjectToAtlas_" + str(sessionid) MergeSessionSubjectToAtlas = pe.Node(interface=Merge(file_count), run_without_submitting=True, name=MergeSessionSubjectToAtlasName) baw200.connect([(myLocalSegWF, MergeSessionSubjectToAtlas, [('outputspec.outputBinaryLeftAccumben', 'in1'), ('outputspec.outputBinaryLeftCaudate', 'in2'), ('outputspec.outputBinaryLeftPutamen', 'in3'), ('outputspec.outputBinaryLeftGlobus', 'in4'), ('outputspec.outputBinaryLeftThalamus', 'in5'), ('outputspec.outputBinaryLeftHippocampus', 'in6'), ('outputspec.outputBinaryRightAccumben', 'in7'), ('outputspec.outputBinaryRightCaudate', 'in8'), ('outputspec.outputBinaryRightPutamen', 'in9'), ('outputspec.outputBinaryRightGlobus', 'in10'), ('outputspec.outputBinaryRightThalamus', 'in11'), ('outputspec.outputBinaryRightHippocampus', 'in12')]), # (FixWMPartitioningNode, MergeSessionSubjectToAtlas, [('UpdatedPosteriorsList', 'in13')]), (inputsSpec, MergeSessionSubjectToAtlas, [('UpdatedPosteriorsList', 'in13')]), (inputsSpec, MergeSessionSubjectToAtlas, [('t1_average', 'in14')]) ]) if not onlyT1: assert file_count == 15 baw200.connect([(inputsSpec, MergeSessionSubjectToAtlas, [('t2_average', 'in15')])]) LinearSubjectToAtlasANTsApplyTransformsName = 'LinearSubjectToAtlasANTsApplyTransforms_' + str(sessionid) LinearSubjectToAtlasANTsApplyTransforms = pe.MapNode(interface=ants.ApplyTransforms(), iterfield=['input_image'], name=LinearSubjectToAtlasANTsApplyTransformsName) LinearSubjectToAtlasANTsApplyTransforms.inputs.interpolation = 'Linear' baw200.connect([(AtlasToSubjectantsRegistration, LinearSubjectToAtlasANTsApplyTransforms, [('inverse_composite_transform', 'transforms')]), (inputsSpec, LinearSubjectToAtlasANTsApplyTransforms, [('template_t1', 'reference_image')]), (MergeSessionSubjectToAtlas, LinearSubjectToAtlasANTsApplyTransforms, [('out', 'input_image')]) ]) MergeMultiLabelSessionSubjectToAtlasName = "99_MergeMultiLabelSessionSubjectToAtlas_" + str(sessionid) MergeMultiLabelSessionSubjectToAtlas = pe.Node(interface=Merge(2), run_without_submitting=True, name=MergeMultiLabelSessionSubjectToAtlasName) baw200.connect([(inputsSpec, MergeMultiLabelSessionSubjectToAtlas, [('inputLabels', 'in1'), ('inputHeadLabels', 'in2')]) ]) ### This is taking this sessions RF label map back into NAC atlas space. #{ MultiLabelSubjectToAtlasANTsApplyTransformsName = 'MultiLabelSubjectToAtlasANTsApplyTransforms_' + str(sessionid) + '_map' MultiLabelSubjectToAtlasANTsApplyTransforms = pe.MapNode(interface=ants.ApplyTransforms(), iterfield=['input_image'], name=MultiLabelSubjectToAtlasANTsApplyTransformsName) MultiLabelSubjectToAtlasANTsApplyTransforms.inputs.interpolation = 'MultiLabel' baw200.connect([(AtlasToSubjectantsRegistration, MultiLabelSubjectToAtlasANTsApplyTransforms, [('inverse_composite_transform', 'transforms')]), (inputsSpec, MultiLabelSubjectToAtlasANTsApplyTransforms, [('template_t1', 'reference_image')]), (MergeMultiLabelSessionSubjectToAtlas, MultiLabelSubjectToAtlasANTsApplyTransforms, [('out', 'input_image')]) ]) #} ### Now we must take the sessions to THIS SUBJECTS personalized atlas. #{ #} ### Now define where the final organized outputs should go. Subj2Atlas_DSName = "SubjectToAtlas_DS_" + str(sessionid) Subj2Atlas_DS = pe.Node(nio.DataSink(), name=Subj2Atlas_DSName) Subj2Atlas_DS.overwrite = master_config['ds_overwrite'] Subj2Atlas_DS.inputs.base_directory = master_config['resultdir'] Subj2Atlas_DS.inputs.regexp_substitutions = [(r'_LinearSubjectToAtlasANTsApplyTransforms_[^/]*', r'' + sessionid + '/')] baw200.connect([(LinearSubjectToAtlasANTsApplyTransforms, Subj2Atlas_DS, [('output_image', 'SubjectToAtlasWarped.@linear_output_images')])]) Subj2AtlasTransforms_DSName = "SubjectToAtlasTransforms_DS_" + str(sessionid) Subj2AtlasTransforms_DS = pe.Node(nio.DataSink(), name=Subj2AtlasTransforms_DSName) Subj2AtlasTransforms_DS.overwrite = master_config['ds_overwrite'] Subj2AtlasTransforms_DS.inputs.base_directory = master_config['resultdir'] Subj2AtlasTransforms_DS.inputs.regexp_substitutions = [(r'SubjectToAtlasWarped', r'SubjectToAtlasWarped/' + sessionid + '/')] baw200.connect([(AtlasToSubjectantsRegistration, Subj2AtlasTransforms_DS, [('composite_transform', 'SubjectToAtlasWarped.@composite_transform'), ('inverse_composite_transform', 'SubjectToAtlasWarped.@inverse_composite_transform')])]) # baw200.connect([(MultiLabelSubjectToAtlasANTsApplyTransforms, Subj2Atlas_DS, [('output_image', 'SubjectToAtlasWarped.@multilabel_output_images')])]) if master_config['execution']['plugin'] == 'SGE': # for some nodes, the qsub call needs to be modified on the cluster AtlasToSubjectantsRegistration.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], '9000M', 4, hard=False)} SnapShotWriter.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], '1000M', 1, 1, hard=False)} LinearSubjectToAtlasANTsApplyTransforms.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], '1000M', 1, hard=True)} MultiLabelSubjectToAtlasANTsApplyTransforms.plugin_args = {'template': master_config['plugin_args']['template'], 'overwrite': True, 'qsub_args': modify_qsub_args(master_config['queue'], '1000M', 1, hard=True)} return baw200
wf.config['execution'] = {'hash_method': 'timestamp', 'crashdump_dir': os.path.abspath(c.crashLogDirectory)} log_dir = os.path.join(c.outputDirectory, 'logs', 'group_analysis', resource, 'model_%s' % (os.path.basename(model))) try: os.makedirs(log_dir) except: print "log_dir already exist" # enable logging from nipype import config from nipype import logging config.update_config({'logging': {'log_directory': log_dir, 'log_to_file': True}}) # Temporarily disable until solved #logging.update_logging(config) iflogger = logging.getLogger('interface') group_sublist = open(subject_list, 'r') #print >>diag, "> Opened subject list: ", subject_list #print >>diag, "" sublist_items = group_sublist.readlines() input_subject_list = [line.rstrip('\n') for line in sublist_items \
def RunSubjectWorkflow(args): """ .-----------. --- | Session 1 | ---> /project/subjectA/session1/phase/ / *-----------* .-----------. / | Subject A | < *-----------* \ \ .-----------. --- | Session 2 | ---> /project/subjectA/session2/phase/ *-----------* **** Replaces WorkflowT1T2.py **** """ database, start_time, subject, master_config = args assert 'baseline' in master_config['components'] or 'longitudinal' in master_config['components'], "Baseline or Longitudinal is not in WORKFLOW_COMPONENTS!" # HACK: # To avoid a "sqlite3.ProgrammingError: Base Cursor.__init__ not called" error # using multiprocessing.map_async(), re-instantiate database # database.__init__(defaultDBName=database.dbName, subject_list=database.subjectList) # # END HACK import time from nipype import config, logging config.update_config(master_config) # Set universal pipeline options assert config.get('execution', 'plugin') == master_config['execution']['plugin'] # DEBUG # config.enable_debug_mode() # config.set('execution', 'stop_on_first_rerun', 'true') # END DEBUG logging.update_logging(config) import nipype.pipeline.engine as pe import nipype.interfaces.base as nbase import nipype.interfaces.io as nio from nipype.interfaces.utility import IdentityInterface, Function import traits from baw_exp import OpenSubjectDatabase from SessionDB import SessionDB from PipeLineFunctionHelpers import convertToList from atlasNode import MakeAtlasNode from utilities.misc import GenerateSubjectOutputPattern as outputPattern from utilities.misc import GenerateWFName while time.time() < start_time: time.sleep(start_time - time.time() + 1) print "Delaying start for {subject}".format(subject=subject) print("===================== SUBJECT: {0} ===========================".format(subject)) subjectWorkflow = pe.Workflow(name="BAW_StandardWorkup_subject_{0}".format(subject)) subjectWorkflow.base_dir = config.get('logging', 'log_directory') # subjectWorkflow.config['execution']['plugin'] = 'Linear' # Hardcodeded in WorkupT1T2.py - why? # DEBUG # subjectWorkflow.config['execution']['stop_on_first_rerun'] = 'true' # END DEBUG atlasNode = MakeAtlasNode(master_config['atlascache'], 'BAtlas') sessionWorkflow = dict() inputsSpec = dict() sessions = database.getSessionsFromSubject(subject) # print "These are the sessions: ", sessions if 'baseline' in master_config['components']: current_phase = 'baseline' from baseline import create_baseline as create_wkfl elif 'longitudinal' in master_config['components']: current_phase = 'longitudinal' from longitudinal import create_longitudial as create_wkfl for session in sessions: # TODO (future): Replace with iterable inputSpec node and add Function node for getAllFiles() project = database.getProjFromSession(session) pname = "{0}_{1}".format(session, current_phase) # Long node names make graphs a pain to read/print # pname = GenerateWFName(project, subject, session, current_phase) print "Building session pipeline for {0}".format(session) inputsSpec[session] = pe.Node(name='inputspec_{0}'.format(session), interface=IdentityInterface(fields=['T1s', 'T2s', 'PDs', 'FLs', 'OTs'])) inputsSpec[session].inputs.T1s = database.getFilenamesByScantype(session, ['T1-15', 'T1-30']) inputsSpec[session].inputs.T2s = database.getFilenamesByScantype(session, ['T2-15', 'T2-30']) inputsSpec[session].inputs.PDs = database.getFilenamesByScantype(session, ['PD-15', 'PD-30']) inputsSpec[session].inputs.FLs = database.getFilenamesByScantype(session, ['FL-15', 'FL-30']) inputsSpec[session].inputs.OTs = database.getFilenamesByScantype(session, ['OTHER-15', 'OTHER-30']) sessionWorkflow[session] = create_wkfl(project, subject, session, master_config, interpMode='Linear', pipeline_name=pname) subjectWorkflow.connect([(inputsSpec[session], sessionWorkflow[session], [('T1s', 'inputspec.T1s'), ('T2s', 'inputspec.T2s'), ('PDs', 'inputspec.PDs'), ('FLs', 'inputspec.FLs'), ('OTs', 'inputspec.OTHERs'), ]), (atlasNode, sessionWorkflow[session], [('template_landmarks_50Lmks_fcsv', 'inputspec.atlasLandmarkFilename'), ('template_weights_50Lmks_wts', 'inputspec.atlasWeightFilename'), ('LLSModel_50Lmks_hdf5', 'inputspec.LLSModel'), ('T1_50Lmks_mdl', 'inputspec.inputTemplateModel')]), ]) if current_phase == 'baseline': subjectWorkflow.connect([(atlasNode, sessionWorkflow[session], [('template_t1', 'inputspec.template_t1'), ('ExtendedAtlasDefinition_xml', 'inputspec.atlasDefinition')]), ]) else: assert current_phase == 'longitudinal', "Phase value is unknown: {0}".format(current_phase) from utils import run_workflow, print_workflow if False: print_workflow(template, plugin=master_config['execution']['plugin'], dotfilename='template') return run_workflow(template, plugin=master_config['execution']['plugin'], plugin_args=master_config['plugin_args'])
def build_workflow(opts, retval): """ Create the Nipype Workflow that supports the whole execution graph, given the inputs. All the checks and the construction of the workflow are done inside this function that has pickleable inputs and output dictionary (``retval``) to allow isolation using a ``multiprocessing.Process`` that allows fmriprep to enforce a hard-limited memory-scope. """ from subprocess import check_call, CalledProcessError, TimeoutExpired from pkg_resources import resource_filename as pkgrf from shutil import copyfile from nipype import logging, config as ncfg from niworkflows.utils.bids import collect_participants from ..__about__ import __version__ from ..workflows.base import init_fmriprep_wf from ..viz.reports import generate_reports logger = logging.getLogger('nipype.workflow') INIT_MSG = """ Running fMRIPREP version {version}: * BIDS dataset path: {bids_dir}. * Participant list: {subject_list}. * Run identifier: {uuid}. """.format output_spaces = opts.output_space or [] # Validity of some inputs # ERROR check if use_aroma was specified, but the correct template was not if opts.use_aroma and (opts.template != 'MNI152NLin2009cAsym' or 'template' not in output_spaces): output_spaces.append('template') logger.warning( 'Option "--use-aroma" requires functional images to be resampled to MNI space. ' 'The argument "template" has been automatically added to the list of output ' 'spaces (option "--output-space").') if opts.cifti_output and (opts.template != 'MNI152NLin2009cAsym' or 'template' not in output_spaces): output_spaces.append('template') logger.warning( 'Option "--cifti-output" requires functional images to be resampled to MNI space. ' 'The argument "template" has been automatically added to the list of output ' 'spaces (option "--output-space").') # Check output_space if 'template' not in output_spaces and (opts.use_syn_sdc or opts.force_syn): msg = [ 'SyN SDC correction requires T1 to MNI registration, but ' '"template" is not specified in "--output-space" arguments.', 'Option --use-syn will be cowardly dismissed.' ] if opts.force_syn: output_spaces.append('template') msg[1] = ( ' Since --force-syn has been requested, "template" has been added to' ' the "--output-space" list.') logger.warning(' '.join(msg)) # Set up some instrumental utilities run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4()) # First check that bids_dir looks like a BIDS folder bids_dir = os.path.abspath(opts.bids_dir) subject_list = collect_participants( bids_dir, participant_label=opts.participant_label) # Load base plugin_settings from file if --use-plugin if opts.use_plugin is not None: from yaml import load as loadyml with open(opts.use_plugin) as f: plugin_settings = loadyml(f) plugin_settings.setdefault('plugin_args', {}) else: # Defaults plugin_settings = { 'plugin': 'MultiProc', 'plugin_args': { 'raise_insufficient': False, 'maxtasksperchild': 1, } } # Resource management options # Note that we're making strong assumptions about valid plugin args # This may need to be revisited if people try to use batch plugins nthreads = plugin_settings['plugin_args'].get('n_procs') # Permit overriding plugin config with specific CLI options if nthreads is None or opts.nthreads is not None: nthreads = opts.nthreads if nthreads is None or nthreads < 1: nthreads = cpu_count() plugin_settings['plugin_args']['n_procs'] = nthreads if opts.mem_mb: plugin_settings['plugin_args']['memory_gb'] = opts.mem_mb / 1024 omp_nthreads = opts.omp_nthreads if omp_nthreads == 0: omp_nthreads = min(nthreads - 1 if nthreads > 1 else cpu_count(), 8) if 1 < nthreads < omp_nthreads: logger.warning( 'Per-process threads (--omp-nthreads=%d) exceed total ' 'threads (--nthreads/--n_cpus=%d)', omp_nthreads, nthreads) # Set up directories output_dir = op.abspath(opts.output_dir) log_dir = op.join(output_dir, 'fmriprep', 'logs') work_dir = op.abspath(opts.work_dir or 'work') # Set work/ as default # Check and create output and working directories os.makedirs(output_dir, exist_ok=True) os.makedirs(log_dir, exist_ok=True) os.makedirs(work_dir, exist_ok=True) # Nipype config (logs and execution) ncfg.update_config({ 'logging': { 'log_directory': log_dir, 'log_to_file': True }, 'execution': { 'crashdump_dir': log_dir, 'crashfile_format': 'txt', 'get_linked_libs': False, 'stop_on_first_crash': opts.stop_on_first_crash or opts.work_dir is None, }, 'monitoring': { 'enabled': opts.resource_monitor, 'sample_frequency': '0.5', 'summary_append': True, } }) if opts.resource_monitor: ncfg.enable_resource_monitor() retval['return_code'] = 0 retval['plugin_settings'] = plugin_settings retval['bids_dir'] = bids_dir retval['output_dir'] = output_dir retval['work_dir'] = work_dir retval['subject_list'] = subject_list retval['run_uuid'] = run_uuid retval['workflow'] = None # Called with reports only if opts.reports_only: logger.log(25, 'Running --reports-only on participants %s', ', '.join(subject_list)) if opts.run_uuid is not None: run_uuid = opts.run_uuid retval['return_code'] = generate_reports(subject_list, output_dir, work_dir, run_uuid) return retval # Build main workflow logger.log( 25, INIT_MSG(version=__version__, bids_dir=bids_dir, subject_list=subject_list, uuid=run_uuid)) template_out_grid = opts.template_resampling_grid if opts.output_grid_reference is not None: logger.warning( 'Option --output-grid-reference is deprecated, please use ' '--template-resampling-grid') template_out_grid = template_out_grid or opts.output_grid_reference if opts.debug: logger.warning('Option --debug is deprecated and has no effect') retval['workflow'] = init_fmriprep_wf( subject_list=subject_list, task_id=opts.task_id, echo_idx=opts.echo_idx, run_uuid=run_uuid, ignore=opts.ignore, debug=opts.sloppy, low_mem=opts.low_mem, anat_only=opts.anat_only, longitudinal=opts.longitudinal, t2s_coreg=opts.t2s_coreg, omp_nthreads=omp_nthreads, skull_strip_template=opts.skull_strip_template, skull_strip_fixed_seed=opts.skull_strip_fixed_seed, work_dir=work_dir, output_dir=output_dir, bids_dir=bids_dir, freesurfer=opts.run_reconall, output_spaces=output_spaces, template=opts.template, medial_surface_nan=opts.medial_surface_nan, cifti_output=opts.cifti_output, template_out_grid=template_out_grid, hires=opts.hires, use_bbr=opts.use_bbr, bold2t1w_dof=opts.bold2t1w_dof, fmap_bspline=opts.fmap_bspline, fmap_demean=opts.fmap_no_demean, use_syn=opts.use_syn_sdc, force_syn=opts.force_syn, use_aroma=opts.use_aroma, aroma_melodic_dim=opts.aroma_melodic_dimensionality, ignore_aroma_err=opts.ignore_aroma_denoising_errors, ) retval['return_code'] = 0 logs_path = Path(output_dir) / 'fmriprep' / 'logs' boilerplate = retval['workflow'].visit_desc() if boilerplate: (logs_path / 'CITATION.md').write_text(boilerplate) logger.log( 25, 'Works derived from this fMRIPrep execution should ' 'include the following boilerplate:\n\n%s', boilerplate) # Generate HTML file resolving citations cmd = [ 'pandoc', '-s', '--bibliography', pkgrf('fmriprep', 'data/boilerplate.bib'), '--filter', 'pandoc-citeproc', '--metadata', 'pagetitle="fMRIPrep citation boilerplate"', str(logs_path / 'CITATION.md'), '-o', str(logs_path / 'CITATION.html') ] try: check_call(cmd, timeout=10) except (FileNotFoundError, CalledProcessError, TimeoutExpired): logger.warning('Could not generate CITATION.html file:\n%s', ' '.join(cmd)) # Generate LaTex file resolving citations cmd = [ 'pandoc', '-s', '--bibliography', pkgrf('fmriprep', 'data/boilerplate.bib'), '--natbib', str(logs_path / 'CITATION.md'), '-o', str(logs_path / 'CITATION.tex') ] try: check_call(cmd, timeout=10) except (FileNotFoundError, CalledProcessError, TimeoutExpired): logger.warning('Could not generate CITATION.tex file:\n%s', ' '.join(cmd)) else: copyfile(pkgrf('fmriprep', 'data/boilerplate.bib'), (logs_path / 'CITATION.bib')) return retval
def build_workflow(opts, retval): """ Create the Nipype Workflow that supports the whole execution graph, given the inputs. All the checks and the construction of the workflow are done inside this function that has pickleable inputs and output dictionary (``retval``) to allow isolation using a ``multiprocessing.Process`` that allows fmriprep to enforce a hard-limited memory-scope. """ from bids import BIDSLayout from nipype import logging as nlogging, config as ncfg from niworkflows.utils.bids import collect_participants from niworkflows.reports import generate_reports from ..__about__ import __version__ from ..workflows.base import init_fmriprep_wf build_log = nlogging.getLogger('nipype.workflow') INIT_MSG = """ Running fMRIPREP version {version}: * BIDS dataset path: {bids_dir}. * Participant list: {subject_list}. * Run identifier: {uuid}. """.format bids_dir = opts.bids_dir.resolve() output_dir = opts.output_dir.resolve() work_dir = opts.work_dir.resolve() retval['return_code'] = 1 retval['workflow'] = None retval['bids_dir'] = str(bids_dir) retval['output_dir'] = str(output_dir) retval['work_dir'] = str(work_dir) if output_dir == bids_dir: build_log.error( 'The selected output folder is the same as the input BIDS folder. ' 'Please modify the output path (suggestion: %s).', bids_dir / 'derivatives' / ('fmriprep-%s' % __version__.split('+')[0])) retval['return_code'] = 1 return retval output_spaces = parse_spaces(opts) # Set up some instrumental utilities run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4()) retval['run_uuid'] = run_uuid # First check that bids_dir looks like a BIDS folder layout = BIDSLayout(str(bids_dir), validate=False, ignore=("code", "stimuli", "sourcedata", "models", "derivatives", re.compile(r'^\.'))) subject_list = collect_participants( layout, participant_label=opts.participant_label) retval['subject_list'] = subject_list # Load base plugin_settings from file if --use-plugin if opts.use_plugin is not None: from yaml import load as loadyml with open(opts.use_plugin) as f: plugin_settings = loadyml(f) plugin_settings.setdefault('plugin_args', {}) else: # Defaults plugin_settings = { 'plugin': 'MultiProc', 'plugin_args': { 'raise_insufficient': False, 'maxtasksperchild': 1, } } # Resource management options # Note that we're making strong assumptions about valid plugin args # This may need to be revisited if people try to use batch plugins nthreads = plugin_settings['plugin_args'].get('n_procs') # Permit overriding plugin config with specific CLI options if nthreads is None or opts.nthreads is not None: nthreads = opts.nthreads if nthreads is None or nthreads < 1: nthreads = cpu_count() plugin_settings['plugin_args']['n_procs'] = nthreads if opts.mem_mb: plugin_settings['plugin_args']['memory_gb'] = opts.mem_mb / 1024 omp_nthreads = opts.omp_nthreads if omp_nthreads == 0: omp_nthreads = min(nthreads - 1 if nthreads > 1 else cpu_count(), 8) if 1 < nthreads < omp_nthreads: build_log.warning( 'Per-process threads (--omp-nthreads=%d) exceed total ' 'threads (--nthreads/--n_cpus=%d)', omp_nthreads, nthreads) retval['plugin_settings'] = plugin_settings # Set up directories log_dir = output_dir / 'fmriprep' / 'logs' # Check and create output and working directories output_dir.mkdir(exist_ok=True, parents=True) log_dir.mkdir(exist_ok=True, parents=True) work_dir.mkdir(exist_ok=True, parents=True) # Nipype config (logs and execution) ncfg.update_config({ 'logging': { 'log_directory': str(log_dir), 'log_to_file': True }, 'execution': { 'crashdump_dir': str(log_dir), 'crashfile_format': 'txt', 'get_linked_libs': False, 'stop_on_first_crash': opts.stop_on_first_crash, }, 'monitoring': { 'enabled': opts.resource_monitor, 'sample_frequency': '0.5', 'summary_append': True, } }) if opts.resource_monitor: ncfg.enable_resource_monitor() # Called with reports only if opts.reports_only: build_log.log(25, 'Running --reports-only on participants %s', ', '.join(subject_list)) if opts.run_uuid is not None: run_uuid = opts.run_uuid retval['run_uuid'] = run_uuid retval['return_code'] = generate_reports(subject_list, output_dir, work_dir, run_uuid, packagename='fmriprep') return retval # Build main workflow build_log.log( 25, INIT_MSG(version=__version__, bids_dir=bids_dir, subject_list=subject_list, uuid=run_uuid)) retval['workflow'] = init_fmriprep_wf( anat_only=opts.anat_only, aroma_melodic_dim=opts.aroma_melodic_dimensionality, bold2t1w_dof=opts.bold2t1w_dof, cifti_output=opts.cifti_output, debug=opts.sloppy, dummy_scans=opts.dummy_scans, echo_idx=opts.echo_idx, err_on_aroma_warn=opts.error_on_aroma_warnings, fmap_bspline=opts.fmap_bspline, fmap_demean=opts.fmap_no_demean, force_syn=opts.force_syn, freesurfer=opts.run_reconall, hires=opts.hires, ignore=opts.ignore, layout=layout, longitudinal=opts.longitudinal, low_mem=opts.low_mem, medial_surface_nan=opts.medial_surface_nan, omp_nthreads=omp_nthreads, output_dir=str(output_dir), output_spaces=output_spaces, run_uuid=run_uuid, regressors_all_comps=opts.return_all_components, regressors_fd_th=opts.fd_spike_threshold, regressors_dvars_th=opts.dvars_spike_threshold, skull_strip_fixed_seed=opts.skull_strip_fixed_seed, skull_strip_template=opts.skull_strip_template, subject_list=subject_list, t2s_coreg=opts.t2s_coreg, task_id=opts.task_id, use_aroma=opts.use_aroma, use_bbr=opts.use_bbr, use_syn=opts.use_syn_sdc, work_dir=str(work_dir), ) retval['return_code'] = 0 logs_path = Path(output_dir) / 'fmriprep' / 'logs' boilerplate = retval['workflow'].visit_desc() if boilerplate: citation_files = { ext: logs_path / ('CITATION.%s' % ext) for ext in ('bib', 'tex', 'md', 'html') } # To please git-annex users and also to guarantee consistency # among different renderings of the same file, first remove any # existing one for citation_file in citation_files.values(): try: citation_file.unlink() except FileNotFoundError: pass citation_files['md'].write_text(boilerplate) build_log.log( 25, 'Works derived from this fMRIPrep execution should ' 'include the following boilerplate:\n\n%s', boilerplate) return retval
def test_masking(input_fname, expected_fname): """Check for regressions in masking.""" from nipype import config as ncfg basename = Path(input_fname[0]).name dsname = Path(expected_fname).parent.name # Reconstruct base_fname from above reports_dir = Path(os.getenv("FMRIPREP_REGRESSION_REPORTS", "")) newpath = reports_dir / dsname newpath.mkdir(parents=True, exist_ok=True) # Nipype config (logs and execution) ncfg.update_config({"execution": { "crashdump_dir": str(newpath), }}) wf = pe.Workflow( name=basename.replace("_bold.nii.gz", "").replace("-", "_")) base_dir = os.getenv("CACHED_WORK_DIRECTORY") if base_dir: base_dir = Path(base_dir) / dsname base_dir.mkdir(parents=True, exist_ok=True) wf.base_dir = str(base_dir) epi_reference_wf = init_epi_reference_wf(omp_nthreads=os.cpu_count(), auto_bold_nss=True) epi_reference_wf.inputs.inputnode.in_files = input_fname enhance_and_skullstrip_bold_wf = init_enhance_and_skullstrip_bold_wf() out_fname = fname_presuffix(Path(expected_fname).name, suffix=".svg", use_ext=False, newpath=str(newpath)) mask_diff_plot = pe.Node(ROIsPlot(colors=["limegreen"], levels=[0.5]), name="mask_diff_plot") mask_diff_plot.always_run = True mask_diff_plot.inputs.in_mask = expected_fname mask_diff_plot.inputs.out_report = out_fname # fmt:off wf.connect([ (epi_reference_wf, enhance_and_skullstrip_bold_wf, [("outputnode.epi_ref_file", "inputnode.in_file")]), (enhance_and_skullstrip_bold_wf, mask_diff_plot, [ ("outputnode.bias_corrected_file", "in_file"), ("outputnode.mask_file", "in_rois"), ]), ]) res = wf.run(plugin="MultiProc") combine_masks = [ node for node in res.nodes if node.name.endswith("combine_masks") ][0] overlap = symmetric_overlap(expected_fname, combine_masks.result.outputs.out_file) mask_dir = reports_dir / "fmriprep_bold_mask" / dsname mask_dir.mkdir(parents=True, exist_ok=True) copyfile( combine_masks.result.outputs.out_file, str(mask_dir / Path(expected_fname).name), copy=True, ) assert overlap > 0.95, input_fname
###################### # WF ###################### wd_dir = '/scr/kansas1/data/lsd-lemon/lemon_wd_meanDist_%s' % distype ds_dir = '/scr/kansas1/data/lsd-lemon/lemon_results_meanDist_%s' % distype wf = Workflow(name='distconnect_meanDist_%s' % distype) wf.base_dir = os.path.join(wd_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={'stop_on_first_crash': False, 'remove_unnecessary_outputs': False, 'job_finished_timeout': 120}) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join(wd_dir, 'crash') ds = Node(nio.DataSink(base_directory=ds_dir), name='ds') ###################### # GET DATA ###################### # SUBJECTS ITERATOR subjects_infosource = Node(util.IdentityInterface(fields=['subject_id']), name='subjects_infosource') subjects_infosource.iterables = ('subject_id', subjects_list) run_mean_dist = Node(util.Function(input_names=['sub'], output_names=[],
def calc_local_metrics( preprocessed_data_dir, subject_id, parcellations_dict, bp_freq_list, fd_thresh, working_dir, ds_dir, use_n_procs, plugin_name, ): import os from nipype import config from nipype.pipeline.engine import Node, Workflow, MapNode import nipype.interfaces.utility as util import nipype.interfaces.io as nio import nipype.interfaces.fsl as fsl import utils as calc_metrics_utils ##################################### # GENERAL SETTINGS ##################################### fsl.FSLCommand.set_default_output_type("NIFTI_GZ") wf = Workflow(name="LeiCA_LIFE_metrics") wf.base_dir = os.path.join(working_dir) nipype_cfg = dict( logging=dict(workflow_level="DEBUG"), execution={"stop_on_first_crash": True, "remove_unnecessary_outputs": True, "job_finished_timeout": 15}, ) config.update_config(nipype_cfg) wf.config["execution"]["crashdump_dir"] = os.path.join(working_dir, "crash") ds = Node(nio.DataSink(base_directory=ds_dir), name="ds") ds.inputs.regexp_substitutions = [ ("MNI_resampled_brain_mask_calc.nii.gz", "falff.nii.gz"), ("residual_filtered_3dT.nii.gz", "alff.nii.gz"), ("_parcellation_", ""), ("_bp_freqs_", "bp_"), ] ##################### # ITERATORS ##################### # PARCELLATION ITERATOR parcellation_infosource = Node(util.IdentityInterface(fields=["parcellation"]), name="parcellation_infosource") parcellation_infosource.iterables = ("parcellation", parcellations_dict.keys()) bp_filter_infosource = Node(util.IdentityInterface(fields=["bp_freqs"]), name="bp_filter_infosource") bp_filter_infosource.iterables = ("bp_freqs", bp_freq_list) selectfiles = Node( nio.SelectFiles( { "parcellation_time_series": "{subject_id}/con_mat/parcellated_time_series/bp_{bp_freqs}/{parcellation}/parcellation_time_series.npy" }, base_directory=preprocessed_data_dir, ), name="selectfiles", ) selectfiles.inputs.subject_id = subject_id wf.connect(parcellation_infosource, "parcellation", selectfiles, "parcellation") wf.connect(bp_filter_infosource, "bp_freqs", selectfiles, "bp_freqs") fd_file = Node( nio.SelectFiles({"fd_p": "{subject_id}/QC/FD_P_ts"}, base_directory=preprocessed_data_dir), name="fd_file" ) fd_file.inputs.subject_id = subject_id ############## ## CON MATS ############## ############## ## extract ts ############## get_good_trs = Node( util.Function( input_names=["fd_file", "fd_thresh"], output_names=["good_trs", "fd_scrubbed_file"], function=calc_metrics_utils.get_good_trs, ), name="get_good_trs", ) wf.connect(fd_file, "fd_p", get_good_trs, "fd_file") get_good_trs.inputs.fd_thresh = fd_thresh parcellated_ts_scrubbed = Node( util.Function( input_names=["parcellation_time_series_file", "good_trs"], output_names=["parcellation_time_series_scrubbed"], function=calc_metrics_utils.parcellation_time_series_scrubbing, ), name="parcellated_ts_scrubbed", ) wf.connect(selectfiles, "parcellation_time_series", parcellated_ts_scrubbed, "parcellation_time_series_file") wf.connect(get_good_trs, "good_trs", parcellated_ts_scrubbed, "good_trs") ############## ## get conmat ############## con_mat = Node( util.Function( input_names=["in_data", "extraction_method"], output_names=["matrix", "matrix_file"], function=calc_metrics_utils.calculate_connectivity_matrix, ), name="con_mat", ) con_mat.inputs.extraction_method = "correlation" wf.connect(parcellated_ts_scrubbed, "parcellation_time_series_scrubbed", con_mat, "in_data") ############## ## ds ############## wf.connect(get_good_trs, "fd_scrubbed_file", ds, "QC.@fd_scrubbed_file") fd_str = ("%.1f" % fd_thresh).replace(".", "_") wf.connect(con_mat, "matrix_file", ds, "con_mat.matrix_scrubbed_%s.@mat" % fd_str) # wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') # wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') # wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == "CondorDAGMan": wf.run(plugin=plugin_name, plugin_args={"initial_specs": "request_memory = 1500"}) if plugin_name == "MultiProc": wf.run(plugin=plugin_name, plugin_args={"n_procs": use_n_procs})
from nipype import config import clinica.pipelines.engine as cpe # Use hash instead of parameters for iterables folder names # Otherwise path will be too long and generate OSError cfg = dict(execution={"parameterize_dirs": False}) config.update_config(cfg) class PETVolume(cpe.Pipeline): """PETVolume - Volume-based processing of PET images using SPM. Returns: A clinica pipeline object containing the PETVolume pipeline. """ def check_pipeline_parameters(self): """Check pipeline parameters.""" from clinica.utils.atlas import PET_VOLUME_ATLASES from clinica.utils.group import check_group_label self.parameters.setdefault("group_label", None) check_group_label(self.parameters["group_label"]) if "acq_label" not in self.parameters.keys(): raise KeyError( "Missing compulsory acq_label key in pipeline parameter.") self.parameters.setdefault("pvc_psf_tsv", None) self.parameters.setdefault("mask_tissues", [1, 2, 3]) self.parameters.setdefault("mask_threshold", 0.3) self.parameters.setdefault("pvc_mask_tissues", [1, 2, 3])
def preprocessing_pipeline(cfg): import os from nipype import config from nipype.pipeline.engine import Node, Workflow import nipype.interfaces.utility as util import nipype.interfaces.io as nio import nipype.interfaces.fsl as fsl import nipype.interfaces.freesurfer as freesurfer # LeiCA modules from utils import zip_and_save_running_scripts from preprocessing.rsfMRI_preprocessing import create_rsfMRI_preproc_pipeline from preprocessing.converter import create_converter_structural_pipeline, create_converter_functional_pipeline, \ create_converter_diffusion_pipeline # INPUT PARAMETERS dicom_dir = cfg['dicom_dir'] working_dir = cfg['working_dir'] freesurfer_dir = cfg['freesurfer_dir'] template_dir = cfg['template_dir'] script_dir = cfg['script_dir'] ds_dir = cfg['ds_dir'] subject_id = cfg['subject_id'] TR_list = cfg['TR_list'] vols_to_drop = cfg['vols_to_drop'] lp_cutoff_freq = cfg['lp_cutoff_freq'] hp_cutoff_freq = cfg['hp_cutoff_freq'] use_fs_brainmask = cfg['use_fs_brainmask'] use_n_procs = cfg['use_n_procs'] plugin_name = cfg['plugin_name'] ##################################### # GENERAL SETTINGS ##################################### fsl.FSLCommand.set_default_output_type('NIFTI_GZ') freesurfer.FSCommand.set_default_subjects_dir(freesurfer_dir) wf = Workflow(name='LeiCA_resting') wf.base_dir = os.path.join(working_dir) nipype_cfg = dict(logging=dict(workflow_level='DEBUG'), execution={ 'stop_on_first_crash': True, 'remove_unnecessary_outputs': True, 'job_finished_timeout': 120 }) config.update_config(nipype_cfg) wf.config['execution']['crashdump_dir'] = os.path.join( working_dir, 'crash') ds = Node(nio.DataSink(base_directory=ds_dir), name='ds') ##################################### # SET ITERATORS ##################################### # GET SCAN TR_ID ITERATOR scan_infosource = Node(util.IdentityInterface(fields=['TR_id']), name='scan_infosource') scan_infosource.iterables = ('TR_id', TR_list) ##################################### # FETCH MRI DATA ##################################### # GET LATERAL VENTRICLE MASK templates_atlases = { 'lat_ventricle_mask_MNI': 'cpac_image_resources/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz' } selectfiles_templates = Node(nio.SelectFiles(templates_atlases, base_directory=template_dir), name="selectfiles_templates") if not True: # releases 1-6 with 01... format subject_id # GET FUNCTIONAL DATA templates_funct = { 'funct_dicom': '{subject_id}/session_1/RfMRI_*_{TR_id}' } selectfiles_funct = Node(nio.SelectFiles(templates_funct, base_directory=dicom_dir), name="selectfiles_funct") selectfiles_funct.inputs.subject_id = subject_id wf.connect(scan_infosource, 'TR_id', selectfiles_funct, 'TR_id') # GET STRUCTURAL DATA templates_struct = { 't1w_dicom': '{subject_id}/anat', 'dMRI_dicom': '{subject_id}/session_1/DTI_mx_137/*.dcm' } # *.dcm for dMRI as Dcm2nii requires this selectfiles_struct = Node(nio.SelectFiles(templates_struct, base_directory=dicom_dir), name="selectfiles_struct") selectfiles_struct.inputs.subject_id = subject_id else: #startin with release 6 new folder structure templates_funct = { 'funct_dicom': '*/{subject_id}/*_V2/REST_{TR_id}*/*.dcm' } selectfiles_funct = Node(nio.SelectFiles(templates_funct, base_directory=dicom_dir), name="selectfiles_funct") selectfiles_funct.inputs.subject_id = subject_id wf.connect(scan_infosource, 'TR_id', selectfiles_funct, 'TR_id') # GET STRUCTURAL DATA templates_struct = { 't1w_dicom': '*/{subject_id}/*_V2/MPRAGE_SIEMENS_DEFACED*/*.dcm', 'dMRI_dicom': '*/{subject_id}/*_V2/DIFF_137_AP*/*.dcm' } # *.dcm for dMRI as Dcm2nii requires this selectfiles_struct = Node(nio.SelectFiles(templates_struct, base_directory=dicom_dir), name="selectfiles_struct") selectfiles_struct.inputs.subject_id = subject_id ##################################### # COPY RUNNING SCRIPTS ##################################### copy_scripts = Node(util.Function(input_names=['subject_id', 'script_dir'], output_names=['zip_file'], function=zip_and_save_running_scripts), name='copy_scripts') copy_scripts.inputs.script_dir = script_dir copy_scripts.inputs.subject_id = subject_id wf.connect(copy_scripts, 'zip_file', ds, 'scripts') ##################################### # CONVERT DICOMs ##################################### # CONVERT STRUCT 2 NIFTI converter_struct = create_converter_structural_pipeline( working_dir, ds_dir, 'converter_struct') wf.connect(selectfiles_struct, 't1w_dicom', converter_struct, 'inputnode.t1w_dicom') # CONVERT dMRI 2 NIFTI converter_dMRI = create_converter_diffusion_pipeline( working_dir, ds_dir, 'converter_dMRI') wf.connect(selectfiles_struct, 'dMRI_dicom', converter_dMRI, 'inputnode.dMRI_dicom') # CONVERT FUNCT 2 NIFTI converter_funct = create_converter_functional_pipeline( working_dir, ds_dir, 'converter_funct') wf.connect(selectfiles_funct, 'funct_dicom', converter_funct, 'inputnode.epi_dicom') wf.connect(scan_infosource, 'TR_id', converter_funct, 'inputnode.out_format') ##################################### # START RSFMRI PREPROCESSING ANALYSIS ##################################### # rsfMRI PREPROCESSING rsfMRI_preproc = create_rsfMRI_preproc_pipeline(working_dir, freesurfer_dir, ds_dir, use_fs_brainmask, 'rsfMRI_preprocessing') rsfMRI_preproc.inputs.inputnode.vols_to_drop = vols_to_drop rsfMRI_preproc.inputs.inputnode.lp_cutoff_freq = lp_cutoff_freq rsfMRI_preproc.inputs.inputnode.hp_cutoff_freq = hp_cutoff_freq rsfMRI_preproc.inputs.inputnode.subject_id = subject_id wf.connect(converter_struct, 'outputnode.t1w', rsfMRI_preproc, 'inputnode.t1w') wf.connect(converter_funct, 'outputnode.epi', rsfMRI_preproc, 'inputnode.epi') wf.connect(converter_funct, 'outputnode.TR_ms', rsfMRI_preproc, 'inputnode.TR_ms') wf.connect(selectfiles_templates, 'lat_ventricle_mask_MNI', rsfMRI_preproc, 'inputnode.lat_ventricle_mask_MNI') ##################################### # RUN WF ##################################### wf.write_graph(dotfilename=wf.name, graph2use='colored', format='pdf') # 'hierarchical') wf.write_graph(dotfilename=wf.name, graph2use='orig', format='pdf') wf.write_graph(dotfilename=wf.name, graph2use='flat', format='pdf') if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})