def create(self): #, **kwargs): """ Create the nodes and connections for the workflow """ # Preamble csvReader = CSVReader() csvReader.inputs.in_file = self.csv_file.default_value csvReader.inputs.header = self.hasHeader.default_value csvOut = csvReader.run() print("=" * 80) print(csvOut.outputs.__dict__) print("=" * 80) iters = {} label = list(csvOut.outputs.__dict__.keys())[0] result = eval("csvOut.outputs.{0}".format(label)) iters['tests'], iters['trains'] = subsample_crossValidationSet( result, self.sample_size.default_value) # Main event out_fields = ['T1', 'T2', 'Label', 'trainindex', 'testindex'] inputsND = Node(interface=IdentityInterface(fields=out_fields), run_without_submitting=True, name='inputs') inputsND.iterables = [('trainindex', iters['trains']), ('testindex', iters['tests'])] if not self.hasHeader.default_value: inputsND.inputs.T1 = csvOut.outputs.column_0 inputsND.inputs.Label = csvOut.outputs.column_1 inputsND.inputs.T2 = csvOut.outputs.column_2 else: inputsND.inputs.T1 = csvOut.outputs.__dict__['t1'] inputsND.inputs.Label = csvOut.outputs.__dict__['label'] inputsND.inputs.T2 = csvOut.outputs.__dict__['t2'] pass #TODO metaflow = Workflow(name='metaflow') metaflow.config['execution'] = { 'plugin': 'Linear', 'stop_on_first_crash': 'false', 'stop_on_first_rerun': 'false', # This stops at first attempt to rerun, before running, and before deleting previous results. 'hash_method': 'timestamp', 'single_thread_matlab': 'true', # Multi-core 2011a multi-core for matrix multiplication. 'remove_unnecessary_outputs': 'true', 'use_relative_paths': 'false', # relative paths should be on, require hash update when changed. 'remove_node_directories': 'false', # Experimental 'local_hash_check': 'false' } metaflow.add_nodes([inputsND]) """import pdb; pdb.set_trace()""" fusionflow = FusionLabelWorkflow() self.connect([ (metaflow, fusionflow, [('inputs.trainindex', 'trainT1s.index'), ('inputs.T1', 'trainT1s.inlist')]), (metaflow, fusionflow, [('inputs.trainindex', 'trainLabels.index'), ('inputs.Label', 'trainLabels.inlist')]), (metaflow, fusionflow, [('inputs.testindex', 'testT1s.index'), ('inputs.T1', 'testT1s.inlist')]) ])
def create(self): # , **kwargs): """ Create the nodes and connections for the workflow """ # Preamble csvReader = CSVReader() csvReader.inputs.in_file = self.csv_file.default_value csvReader.inputs.header = self.hasHeader.default_value csvOut = csvReader.run() print(("=" * 80)) print((csvOut.outputs.__dict__)) print(("=" * 80)) iters = OrderedDict() label = list(csvOut.outputs.__dict__.keys())[0] result = eval("csvOut.outputs.{0}".format(label)) iters['tests'], iters['trains'] = subsample_crossValidationSet(result, self.sample_size.default_value) # Main event out_fields = ['T1', 'T2', 'Label', 'trainindex', 'testindex'] inputsND = Node(interface=IdentityInterface(fields=out_fields), run_without_submitting=True, name='inputs') inputsND.iterables = [('trainindex', iters['trains']), ('testindex', iters['tests'])] if not self.hasHeader.default_value: inputsND.inputs.T1 = csvOut.outputs.column_0 inputsND.inputs.Label = csvOut.outputs.column_1 inputsND.inputs.T2 = csvOut.outputs.column_2 else: inputsND.inputs.T1 = csvOut.outputs.__dict__['t1'] inputsND.inputs.Label = csvOut.outputs.__dict__['label'] inputsND.inputs.T2 = csvOut.outputs.__dict__['t2'] pass # TODO metaflow = Workflow(name='metaflow') metaflow.config['execution'] = { 'plugin': 'Linear', 'stop_on_first_crash': 'false', 'stop_on_first_rerun': 'false', # This stops at first attempt to rerun, before running, and before deleting previous results. 'hash_method': 'timestamp', 'single_thread_matlab': 'true', # Multi-core 2011a multi-core for matrix multiplication. 'remove_unnecessary_outputs': 'true', 'use_relative_paths': 'false', # relative paths should be on, require hash update when changed. 'remove_node_directories': 'false', # Experimental 'local_hash_check': 'false' } metaflow.add_nodes([inputsND]) """import pdb; pdb.set_trace()""" fusionflow = FusionLabelWorkflow() self.connect( [(metaflow, fusionflow, [('inputs.trainindex', 'trainT1s.index'), ('inputs.T1', 'trainT1s.inlist')]), (metaflow, fusionflow, [('inputs.trainindex', 'trainLabels.index'), ('inputs.Label', 'trainLabels.inlist')]), (metaflow, fusionflow, [('inputs.testindex', 'testT1s.index'), ('inputs.T1', 'testT1s.inlist')]) ])
def create(self): #, **kwargs): """ Create the nodes and connections for the workflow """ # Preamble csvReader = CSVReader() csvReader.inputs.in_file = self.csv_file.default_value csvReader.inputs.header = self.hasHeader.default_value csvOut = csvReader.run() iters = {} label = csvOut.outputs.__dict__.keys()[0] result = eval("csvOut.outputs.{0}".format(label)) iters['tests'], iters['samples'] = sample_test_lists( result, self.sample_size.default_value) # Main event out_fields = ['T1', 'T2', 'Label', 'sampleindex', 'testindex'] inputs = Node(interface=IdentityInterface(fields=out_fields), run_without_submitting=True, name='inputs') inputs.iterables = [('sampleindex', iters['samples']), ('testindex', iters['tests'])] if not self.hasHeader.default_value: inputs.inputs.T1 = csvOut.outputs.column_0 inputs.inputs.Label = csvOut.outputs.column_1 inputs.inputs.T2 = csvOut.outputs.column_2 else: pass #TODO metaflow = Workflow(name='metaflow') metaflow.add_nodes([inputs]) import pdb pdb.set_trace() fusionflow = FusionLabelWorkflow() self.connect([ (metaflow, fusionflow, [('inputs.sampleindex', 'sampleT1s.index'), ('inputs.T1', 'sampleT1s.inlist')]), (metaflow, fusionflow, [('inputs.sampleindex', 'sampleT2s.index'), ('inputs.T2', 'sampleT2s.inlist')]), (metaflow, fusionflow, [('inputs.sampleindex', 'sampleLabels.index'), ('inputs.Label', 'sampleLabels.inlist')]), (metaflow, fusionflow, [('inputs.testindex', 'testT1s.index'), ('inputs.T1', 'testT1s.inlist')]), (metaflow, fusionflow, [('inputs.testindex', 'testT2s.index'), ('inputs.T2', 'testT2s.inlist')]), (metaflow, fusionflow, [('inputs.testindex', 'testLabels.index'), ('inputs.Label', 'testLabels.inlist')]) ])
def init_mriqc_wf(): """Create a multi-subject MRIQC workflow.""" from .. import config workflow = Workflow(name="mriqc_wf") workflow.base_dir = config.execution.work_dir if "bold" in config.workflow.inputs: workflow.add_nodes([fmri_qc_workflow()]) if set(("T1w", "T2w")).intersection( config.workflow.inputs.keys() ): workflow.add_nodes([anat_qc_workflow()]) if not workflow._get_all_nodes(): return None return workflow
def init_mriqc_wf(): """Create a multi-subject MRIQC workflow.""" from mriqc import config # Create parent workflow workflow = Workflow(name="mriqc_wf") workflow.base_dir = config.execution.work_dir # Create fMRI QC workflow if FMRI_KEY in config.workflow.inputs: workflow.add_nodes([fmri_qc_workflow()]) # Create sMRI QC workflow input_keys = config.workflow.inputs.keys() anatomical_flag = any(key in input_keys for key in ANATOMICAL_KEYS) if anatomical_flag: workflow.add_nodes([anat_qc_workflow()]) # Return non-empty workflow, else None if workflow._get_all_nodes(): return workflow
def create_workflow(sources, subjects, basedir=None): import os.path as osp import tempfile from nipype.pipeline.engine import Workflow, Node if len(sources) != len(subjects): raise Exception('Input files and subjects should be of equal size.') wf_name = 'spm12_%s'%subjects[0] if len(subjects) == 1 else 'spm12' if len(sources) == 1 and basedir is None: wf_basedir = osp.dirname(sources[0]) elif not basedir is None: wf_basedir = basedir else: wf_basedir = tempfile.mkdtemp() w = Workflow(wf_name, base_dir = wf_basedir) nodes = [] for subject, source in zip(subjects, sources): nodes.extend(create_nodes(source, subject)) w.add_nodes(nodes) for i in range(0, len(nodes), 3): w.connect(nodes[i], 'coregistered_source', nodes[i+1], 'source') w.connect(nodes[i+1], 'coregistered_source', nodes[i+2], 'channel_files') return w
def main(): """Entry point""" from nipype import config as ncfg from nipype.pipeline.engine import Workflow from mriqc import DEFAULTS from mriqc.utils.bids import collect_bids_data from mriqc.workflows.core import build_workflow # from mriqc.reports.utils import check_reports parser = ArgumentParser(description='MRI Quality Control', formatter_class=RawTextHelpFormatter) parser.add_argument('-v', '--version', action='version', version='mriqc v{}'.format(__version__)) parser.add_argument('bids_dir', action='store', help='The directory with the input dataset ' 'formatted according to the BIDS standard.') parser.add_argument( 'output_dir', action='store', help='The directory where the output files ' 'should be stored. If you are running group level analysis ' 'this folder should be prepopulated with the results of the' 'participant level analysis.') parser.add_argument( 'analysis_level', action='store', nargs='+', help='Level of the analysis that will be performed. ' 'Multiple participant level analyses can be run independently ' '(in parallel) using the same output_dir.', choices=['participant', 'group']) parser.add_argument( '--participant_label', '--subject_list', '-S', action='store', help='The label(s) of the participant(s) that should be analyzed. ' 'The label corresponds to sub-<participant_label> from the ' 'BIDS spec (so it does not include "sub-"). If this parameter ' 'is not provided all subjects should be analyzed. Multiple ' 'participants can be specified with a space separated list.', nargs="*") g_input = parser.add_argument_group('mriqc specific inputs') g_input.add_argument('-m', '--modalities', action='store', nargs='*', choices=['T1w', 'bold', 'T2w'], default=['T1w', 'bold', 'T2w']) g_input.add_argument('-s', '--session-id', action='store') g_input.add_argument('-r', '--run-id', action='store') g_input.add_argument('--nthreads', action='store', type=int, help='number of threads') g_input.add_argument('--n_procs', action='store', default=0, type=int, help='number of threads') g_input.add_argument('--mem_gb', action='store', default=0, type=int, help='available total memory') g_input.add_argument('--write-graph', action='store_true', default=False, help='Write workflow graph.') g_input.add_argument('--dry-run', action='store_true', default=False, help='Do not run the workflow.') g_input.add_argument('--use-plugin', action='store', default=None, help='nipype plugin configuration file') g_input.add_argument('--testing', action='store_true', default=False, help='use testing settings for a minimal footprint') g_input.add_argument( '--hmc-afni', action='store_true', default=True, help='Use ANFI 3dvolreg for head motion correction (HMC)') g_input.add_argument( '--hmc-fsl', action='store_true', default=False, help='Use FSL MCFLIRT for head motion correction (HMC)') g_input.add_argument( '-f', '--float32', action='store_true', default=DEFAULTS['float32'], help= "Cast the input data to float32 if it's represented in higher precision " "(saves space and improves perfomance)") g_input.add_argument('--fft-spikes-detector', action='store_true', default=False, help='Turn on FFT based spike detector (slow).') g_outputs = parser.add_argument_group('mriqc specific outputs') g_outputs.add_argument('-w', '--work-dir', action='store', default=op.join(os.getcwd(), 'work')) g_outputs.add_argument('--report-dir', action='store') g_outputs.add_argument('--verbose-reports', default=False, action='store_true') # ANTs options g_ants = parser.add_argument_group( 'specific settings for ANTs registrations') g_ants.add_argument( '--ants-nthreads', action='store', type=int, default=DEFAULTS['ants_nthreads'], help='number of threads that will be set in ANTs processes') g_ants.add_argument('--ants-settings', action='store', help='path to JSON file with settings for ANTS') # AFNI head motion correction settings g_afni = parser.add_argument_group( 'specific settings for AFNI head motion correction') g_afni.add_argument( '--deoblique', action='store_true', default=False, help='Deoblique the functional scans during head motion ' 'correction preprocessing') g_afni.add_argument( '--despike', action='store_true', default=False, help='Despike the functional scans during head motion correction ' 'preprocessing') g_afni.add_argument( '--start-idx', action='store', type=int, help='Initial volume in functional timeseries that should be ' 'considered for preprocessing') g_afni.add_argument( '--stop-idx', action='store', type=int, help='Final volume in functional timeseries that should be ' 'considered for preprocessing') g_afni.add_argument('--correct-slice-timing', action='store_true', default=False, help='Perform slice timing correction') opts = parser.parse_args() # Build settings dict bids_dir = op.abspath(opts.bids_dir) # Number of processes n_procs = 0 if opts.nthreads is not None: MRIQC_LOG.warn('Option --nthreads has been deprecated in mriqc 0.8.8. ' 'Please use --n_procs instead.') n_procs = opts.nthreads if opts.n_procs is not None: n_procs = opts.n_procs # Check physical memory total_memory = opts.mem_gb if total_memory < 0: try: from psutil import virtual_memory total_memory = virtual_memory().total // (1024**3) + 1 except ImportError: MRIQC_LOG.warn( 'Total physical memory could not be estimated, using %d' 'GB as default', DEFAULT_MEM_GB) total_memory = DEFAULT_MEM_GB if total_memory > 0: av_procs = total_memory // 4 if av_procs < 1: MRIQC_LOG.warn( 'Total physical memory is less than 4GB, memory allocation' ' problems are likely to occur.') n_procs = 1 elif n_procs > av_procs: n_procs = av_procs settings = { 'bids_dir': bids_dir, 'write_graph': opts.write_graph, 'testing': opts.testing, 'hmc_afni': opts.hmc_afni, 'hmc_fsl': opts.hmc_fsl, 'fft_spikes_detector': opts.fft_spikes_detector, 'n_procs': n_procs, 'ants_nthreads': opts.ants_nthreads, 'output_dir': op.abspath(opts.output_dir), 'work_dir': op.abspath(opts.work_dir), 'verbose_reports': opts.verbose_reports or opts.testing, 'float32': opts.float32 } if opts.hmc_afni: settings['deoblique'] = opts.deoblique settings['despike'] = opts.despike settings['correct_slice_timing'] = opts.correct_slice_timing if opts.start_idx: settings['start_idx'] = opts.start_idx if opts.stop_idx: settings['stop_idx'] = opts.stop_idx if opts.ants_settings: settings['ants_settings'] = opts.ants_settings log_dir = op.join(settings['output_dir'], 'logs') analysis_levels = opts.analysis_level if opts.participant_label is None: analysis_levels.append('group') analysis_levels = list(set(analysis_levels)) if len(analysis_levels) > 2: raise RuntimeError('Error parsing analysis levels, got "%s"' % ', '.join(analysis_levels)) settings['report_dir'] = opts.report_dir if not settings['report_dir']: settings['report_dir'] = op.join(settings['output_dir'], 'reports') check_folder(settings['output_dir']) if 'participant' in analysis_levels: check_folder(settings['work_dir']) check_folder(log_dir) check_folder(settings['report_dir']) # Set nipype config ncfg.update_config({ 'logging': { 'log_directory': log_dir, 'log_to_file': True }, 'execution': { 'crashdump_dir': log_dir } }) plugin_settings = {'plugin': 'Linear'} if opts.use_plugin is not None: from yaml import load as loadyml with open(opts.use_plugin) as pfile: plugin_settings = loadyml(pfile) else: # Setup multiprocessing if settings['n_procs'] == 0: settings['n_procs'] = 1 max_parallel_ants = cpu_count() // settings['ants_nthreads'] if max_parallel_ants > 1: settings['n_procs'] = max_parallel_ants if settings['n_procs'] > 1: plugin_settings['plugin'] = 'MultiProc' plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']} MRIQC_LOG.info( 'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s', __version__, ', '.join(analysis_levels), opts.participant_label, settings) # Process data types modalities = opts.modalities dataset = collect_bids_data(settings['bids_dir'], participant_label=opts.participant_label) # Set up participant level if 'participant' in analysis_levels: workflow = Workflow(name='workflow_enumerator') workflow.base_dir = settings['work_dir'] wf_list = [] for mod in modalities: if not dataset[mod]: MRIQC_LOG.warn('No %s scans were found in %s', mod, settings['bids_dir']) continue wf_list.append(build_workflow(dataset[mod], mod, settings=settings)) if wf_list: workflow.add_nodes(wf_list) if not opts.dry_run: workflow.run(**plugin_settings) else: raise RuntimeError( 'Error reading BIDS directory (%s), or the dataset is not ' 'BIDS-compliant.' % settings['bids_dir']) # Set up group level if 'group' in analysis_levels: from mriqc.reports import group_html from mriqc.utils.misc import generate_csv, generate_pred reports_dir = check_folder(op.join(settings['output_dir'], 'reports')) derivatives_dir = op.join(settings['output_dir'], 'derivatives') n_group_reports = 0 for mod in modalities: dataframe, out_csv = generate_csv(derivatives_dir, settings['output_dir'], mod) # If there are no iqm.json files, nothing to do. if dataframe is None: MRIQC_LOG.warn( 'No IQM-JSON files were found for the %s data type in %s. The group-level ' 'report was not generated.', mod, derivatives_dir) continue MRIQC_LOG.info('Summary CSV table for the %s data generated (%s)', mod, out_csv) out_pred = generate_pred(derivatives_dir, settings['output_dir'], mod) if out_pred is not None: MRIQC_LOG.info( 'Predicted QA CSV table for the %s data generated (%s)', mod, out_pred) out_html = op.join(reports_dir, mod + '_group.html') group_html(out_csv, mod, csv_failed=op.join(settings['output_dir'], 'failed_' + mod + '.csv'), out_file=out_html) MRIQC_LOG.info('Group-%s report generated (%s)', mod, out_html) n_group_reports += 1 if n_group_reports == 0: raise Exception( "No data found. No group level reports were generated.")
def create(self): # , **kwargs): """ Create the nodes and connections for the workflow """ # Preamble csvReader = CSVReader() csvReader.inputs.in_file = self.csv_file.default_value csvReader.inputs.header = self.hasHeader.default_value csvOut = csvReader.run() print(("=" * 80)) print((csvOut.outputs.__dict__)) print(("=" * 80)) iters = OrderedDict() label = list(csvOut.outputs.__dict__.keys())[0] result = eval("csvOut.outputs.{0}".format(label)) iters["tests"], iters["trains"] = sample_crossvalidation_set( result, self.sample_size.default_value ) # Main event out_fields = ["T1", "T2", "Label", "trainindex", "testindex"] inputsND = Node( interface=IdentityInterface(fields=out_fields), run_without_submitting=True, name="inputs", ) inputsND.iterables = [ ("trainindex", iters["trains"]), ("testindex", iters["tests"]), ] if not self.hasHeader.default_value: inputsND.inputs.T1 = csvOut.outputs.column_0 inputsND.inputs.Label = csvOut.outputs.column_1 inputsND.inputs.T2 = csvOut.outputs.column_2 else: inputsND.inputs.T1 = csvOut.outputs.__dict__["t1"] inputsND.inputs.Label = csvOut.outputs.__dict__["label"] inputsND.inputs.T2 = csvOut.outputs.__dict__["t2"] pass # TODO metaflow = Workflow(name="metaflow") metaflow.config["execution"] = { "plugin": "Linear", "stop_on_first_crash": "false", "stop_on_first_rerun": "false", # This stops at first attempt to rerun, before running, and before deleting previous results. "hash_method": "timestamp", "single_thread_matlab": "true", # Multi-core 2011a multi-core for matrix multiplication. "remove_unnecessary_outputs": "true", "use_relative_paths": "false", # relative paths should be on, require hash update when changed. "remove_node_directories": "false", # Experimental "local_hash_check": "false", } metaflow.add_nodes([inputsND]) """import pdb; pdb.set_trace()""" fusionflow = FusionLabelWorkflow() self.connect( [ ( metaflow, fusionflow, [ ("inputs.trainindex", "trainT1s.index"), ("inputs.T1", "trainT1s.inlist"), ], ), ( metaflow, fusionflow, [ ("inputs.trainindex", "trainLabels.index"), ("inputs.Label", "trainLabels.inlist"), ], ), ( metaflow, fusionflow, [ ("inputs.testindex", "testT1s.index"), ("inputs.T1", "testT1s.inlist"), ], ), ] )
def init_mriqc(opts, retval): """Build the workflow enumerator""" from bids.grabbids import BIDSLayout from nipype import config as ncfg from nipype.pipeline.engine import Workflow from ..utils.bids import collect_bids_data from ..workflows.core import build_workflow retval['workflow'] = None retval['plugin_settings'] = None # Build settings dict bids_dir = Path(opts.bids_dir).expanduser() output_dir = Path(opts.output_dir).expanduser() # Number of processes n_procs = opts.n_procs or cpu_count() settings = { 'bids_dir': bids_dir.resolve(), 'output_dir': output_dir.resolve(), 'work_dir': opts.work_dir.expanduser().resolve(), 'write_graph': opts.write_graph, 'n_procs': n_procs, 'testing': opts.testing, 'hmc_afni': opts.hmc_afni, 'hmc_fsl': opts.hmc_fsl, 'fft_spikes_detector': opts.fft_spikes_detector, 'ants_nthreads': opts.ants_nthreads, 'ants_float': opts.ants_float, 'verbose_reports': opts.verbose_reports or opts.testing, 'float32': opts.float32, 'ica': opts.ica, 'no_sub': opts.no_sub, 'email': opts.email, 'fd_thres': opts.fd_thres, 'webapi_url': opts.webapi_url, 'webapi_port': opts.webapi_port, 'upload_strict': opts.upload_strict, } if opts.hmc_afni: settings['deoblique'] = opts.deoblique settings['despike'] = opts.despike settings['correct_slice_timing'] = opts.correct_slice_timing if opts.start_idx: settings['start_idx'] = opts.start_idx if opts. stop_idx: settings['stop_idx'] = opts.stop_idx if opts.ants_settings: settings['ants_settings'] = opts.ants_settings if opts.dsname: settings['dataset_name'] = opts.dsname log_dir = settings['output_dir'] / 'logs' # Create directories log_dir.mkdir(parents=True, exist_ok=True) settings['work_dir'].mkdir(parents=True, exist_ok=True) # Set nipype config ncfg.update_config({ 'logging': {'log_directory': str(log_dir), 'log_to_file': True}, 'execution': { 'crashdump_dir': str(log_dir), 'crashfile_format': 'txt', 'resource_monitor': opts.profile}, }) # Plugin configuration plugin_settings = {} if n_procs == 1: plugin_settings['plugin'] = 'Linear' if settings['ants_nthreads'] == 0: settings['ants_nthreads'] = 1 else: plugin_settings['plugin'] = 'MultiProc' plugin_settings['plugin_args'] = {'n_procs': n_procs} if opts.mem_gb: plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb if settings['ants_nthreads'] == 0: # always leave one extra thread for non ANTs work, # don't use more than 8 threads - the speed up is minimal settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8) # Overwrite options if --use-plugin provided if opts.use_plugin and opts.use_plugin.exists(): from yaml import load as loadyml with opts.use_plugin.open() as pfile: plugin_settings.update(loadyml(pfile)) # Process data types modalities = opts.modalities layout = BIDSLayout(str(settings['bids_dir']), exclude=['derivatives', 'sourcedata']) dataset = collect_bids_data( layout, participant_label=opts.participant_label, session=opts.session_id, run=opts.run_id, task=opts.task_id, bids_type=modalities, ) workflow = Workflow(name='workflow_enumerator') workflow.base_dir = settings['work_dir'] wf_list = [] subject_list = [] for mod in modalities: if dataset[mod]: wf_list.append(build_workflow(dataset[mod], mod, settings=settings)) subject_list += dataset[mod] retval['subject_list'] = subject_list if not wf_list: retval['return_code'] = 1 return retval workflow.add_nodes(wf_list) retval['plugin_settings'] = plugin_settings retval['workflow'] = workflow retval['return_code'] = 0 return retval
(subject_id))[0] fssubjects_dir = "/scr/adenauer1/internet_study/freesurfer/" output_file = "/scr/adenauer1/internet_study/results/%s/report.pdf" % ( subject_id) report = Node(Function(input_names=[ 'subject_id', 'tsnr_file', 'realignment_parameters_file', 'mean_epi_file', 'mask_file', 'reg_file', 'fssubjects_dir', 'similarity_distribution', 'mean_FD_distribution', 'tsnr_distributions', 'output_file' ], output_names=['out'], function=create_report), name="report_%s" % (subject_id).replace(".", "_")) report.inputs.subject_id = subject_id report.inputs.tsnr_file = tsnr_file report.inputs.realignment_parameters_file = realignment_parameters_file report.inputs.mean_epi_file = mean_epi_file report.inputs.mask_file = mask_file report.inputs.reg_file = reg_file report.inputs.fssubjects_dir = fssubjects_dir report.inputs.similarity_distribution = similarity_distribution report.inputs.mean_FD_distribution = mean_FD_distribution report.inputs.tsnr_distributions = tsnr_distributions report.inputs.output_file = output_file report.plugin_args = {'override_specs': 'request_memory = 4000'} wf.add_nodes([report]) wf.run(plugin="CondorDAGMan")
# In[48]: # Establish input/output stream infosource = Node(IdentityInterface(fields=['subject_id']), name = "infosource") infosource.iterables = [('subject_id', subject_list_test)] lhtemplate_files = opj('lhtemplate*.nii.gz') label_files = opj('{subject_id}-lab.nii.gz') t1_files = opj('{subject_id}-t1-mask.nii.gz') t2_files = opj('{subject_id}-t2s-bfc-mask.nii.gz') templates = {'lhtemplate': lhtemplate_files, 'label_files': label_files, 't1_files': t1_files, 't2_files': t2_files,} selectfiles = Node(SelectFiles(templates, base_directory=datadir), name = "selectfiles") # In[55]: # Create pipeline and connect nodes workflow = Workflow(name='normflow') workflow.base_dir = '.' workflow.add_nodes([test_antsreg_rigid]) #workflow.connect([(infosource, selectfiles, [('subject_id', 'subject_id')]), # (selectfiles, test_antsreg_rigid, [('lhtemplate','moving_image')]),]) workflow.write_graph() workflow.run()
# CHECK IF ALL FILES EXIST def check_if_out_files_exist(check_file_dict): for file in check_file_dict.values(): if not os.path.exists(file): raise Exception('file missing: %s'%file) check_file_dict = file_dict.copy() check_file_dict.pop('report_file') check_if_out_files_exist(check_file_dict) report = Node(util.Function(input_names=['subject_id', 'file_dict', 'df'], output_names=[], function=create_qc_report_pdf), name='report_%s_%s'%(TR, subject_id)) report.inputs.subject_id = subject_id report.inputs.file_dict = file_dict report.inputs.df = df wf.add_nodes([report]) # fixme # ignore warning from np.rank import warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") if plugin_name == 'CondorDAGMan': wf.run(plugin=plugin_name) if plugin_name == 'MultiProc': wf.run(plugin=plugin_name, plugin_args={'n_procs': use_n_procs})
def init_mriqc(opts, retval): """Build the workflow enumerator""" from bids.grabbids import BIDSLayout from nipype import config as ncfg from nipype.pipeline.engine import Workflow from ..utils.bids import collect_bids_data from ..workflows.core import build_workflow retval['workflow'] = None retval['plugin_settings'] = None # Build settings dict bids_dir = Path(opts.bids_dir).expanduser() output_dir = Path(opts.output_dir).expanduser() # Number of processes n_procs = opts.n_procs or cpu_count() settings = { 'bids_dir': bids_dir.resolve(), 'output_dir': output_dir.resolve(), 'work_dir': opts.work_dir.expanduser().resolve(), 'write_graph': opts.write_graph, 'n_procs': n_procs, 'testing': opts.testing, 'hmc_afni': opts.hmc_afni, 'hmc_fsl': opts.hmc_fsl, 'fft_spikes_detector': opts.fft_spikes_detector, 'ants_nthreads': opts.ants_nthreads, 'ants_float': opts.ants_float, 'verbose_reports': opts.verbose_reports or opts.testing, 'float32': opts.float32, 'ica': opts.ica, 'no_sub': opts.no_sub, 'email': opts.email, 'fd_thres': opts.fd_thres, 'webapi_url': opts.webapi_url, 'webapi_port': opts.webapi_port, 'upload_strict': opts.upload_strict, } if opts.hmc_afni: settings['deoblique'] = opts.deoblique settings['despike'] = opts.despike settings['correct_slice_timing'] = opts.correct_slice_timing if opts.start_idx: settings['start_idx'] = opts.start_idx if opts. stop_idx: settings['stop_idx'] = opts.stop_idx if opts.ants_settings: settings['ants_settings'] = opts.ants_settings if opts.dsname: settings['dataset_name'] = opts.dsname log_dir = settings['output_dir'] / 'logs' # Create directories log_dir.mkdir(parents=True, exist_ok=True) settings['work_dir'].mkdir(parents=True, exist_ok=True) # Set nipype config ncfg.update_config({ 'logging': {'log_directory': str(log_dir), 'log_to_file': True}, 'execution': { 'crashdump_dir': str(log_dir), 'crashfile_format': 'txt', 'resource_monitor': opts.profile}, }) # Plugin configuration plugin_settings = {} if n_procs == 1: plugin_settings['plugin'] = 'Linear' if settings['ants_nthreads'] == 0: settings['ants_nthreads'] = 1 else: plugin_settings['plugin'] = 'MultiProc' plugin_settings['plugin_args'] = {'n_procs': n_procs} if opts.mem_gb: plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb if settings['ants_nthreads'] == 0: # always leave one extra thread for non ANTs work, # don't use more than 8 threads - the speed up is minimal settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8) # Overwrite options if --use-plugin provided if opts.use_plugin and opts.use_plugin.exists(): from yaml import load as loadyml with opts.use_plugin.open() as pfile: plugin_settings.update(loadyml(pfile)) # Process data types modalities = opts.modalities layout = BIDSLayout(str(settings['bids_dir']), exclude=['derivatives', 'sourcedata']) dataset = collect_bids_data( layout, participant_label=opts.participant_label, session=opts.session_id, run=opts.run_id, task=opts.task_id, bids_type=modalities, ) workflow = Workflow(name='workflow_enumerator') workflow.base_dir = settings['work_dir'] wf_list = [] subject_list = [] for mod in modalities: if dataset[mod]: wf_list.append(build_workflow(dataset[mod], mod, settings=settings)) subject_list += dataset[mod] retval['subject_list'] = subject_list if not wf_list: retval['return_code'] = 1 return retval workflow.add_nodes(wf_list) retval['plugin_settings'] = plugin_settings retval['workflow'] = workflow retval['return_code'] = 0 return retval
acq_str = "0 -1 0 0.0684" study = "camcan" elif "olm" in subject: acq_str = "0 1 0 {TotalReadoutTime}" study = "olm" else: raise ("Cannot determine study") wfs = [] wfs.append(run_process_dwi(wf_dir, subject, sessions, args, study, prep_pipe="mrtrix", acq_str=acq_str, ants_quick=args.ants_reg_quick)) wf = Workflow(name=subject) wf.base_dir = wf_dir wf.config['execution']['crashdump_dir'] = os.path.join(args.output_dir, "crash") wf.add_nodes(wfs) wf.write_graph(graph2use='colored') try: wf.run(plugin='MultiProc', plugin_args={'n_procs': args.n_cpus}) except: print("Something went wrong") dump_dir = os.path.join(args.output_dir, "crash_dump_wdir", subject) shutil.copytree(os.path.join(wf_dir, subject), dump_dir) print("Copy working directory to " + dump_dir) raise Exception() elif args.analysis_level == "group": output_dir = os.path.join(args.output_dir, "00_group") extracted_dir = os.path.join(args.output_dir, "extracted_metrics") preprocessed_dir = os.path.join(args.output_dir, "dwi_preprocessed")
def main(): """Entry point""" from nipype import config as ncfg from nipype.pipeline.engine import Workflow from mriqc.utils.bids import collect_bids_data from mriqc.workflows.core import build_workflow # Run parser opts = get_parser().parse_args() # Build settings dict bids_dir = op.abspath(opts.bids_dir) # Number of processes n_procs = opts.n_procs settings = { 'bids_dir': bids_dir, 'write_graph': opts.write_graph, 'testing': opts.testing, 'hmc_afni': opts.hmc_afni, 'hmc_fsl': opts.hmc_fsl, 'fft_spikes_detector': opts.fft_spikes_detector, 'n_procs': n_procs, 'ants_nthreads': opts.ants_nthreads, 'output_dir': op.abspath(opts.output_dir), 'work_dir': op.abspath(opts.work_dir), 'verbose_reports': opts.verbose_reports or opts.testing, 'float32': opts.float32, 'ica': opts.ica, 'no_sub': opts.no_sub or opts.testing, 'email': opts.email, 'fd_thres': opts.fd_thres, } if not settings['no_sub']: MRIQC_LOG.warn('Anonymized quality metrics will be submitted' ' to MRIQC\'s metrics repository.' ' Use --no-sub to disable submission.') if opts.hmc_afni: settings['deoblique'] = opts.deoblique settings['despike'] = opts.despike settings['correct_slice_timing'] = opts.correct_slice_timing if opts.start_idx: settings['start_idx'] = opts.start_idx if opts. stop_idx: settings['stop_idx'] = opts.stop_idx if opts.ants_settings: settings['ants_settings'] = opts.ants_settings log_dir = op.join(settings['output_dir'], 'logs') analysis_levels = opts.analysis_level if opts.participant_label is None: analysis_levels.append('group') analysis_levels = list(set(analysis_levels)) if len(analysis_levels) > 2: raise RuntimeError('Error parsing analysis levels, got "%s"' % ', '.join(analysis_levels)) settings['report_dir'] = opts.report_dir if not settings['report_dir']: settings['report_dir'] = op.join(settings['output_dir'], 'reports') check_folder(settings['output_dir']) if 'participant' in analysis_levels: check_folder(settings['work_dir']) check_folder(log_dir) check_folder(settings['report_dir']) # Set nipype config ncfg.update_config({ 'logging': {'log_directory': log_dir, 'log_to_file': True}, 'execution': {'crashdump_dir': log_dir, 'crashfile_format': 'txt'}, }) callback_log_path = None plugin_settings = {'plugin': 'Linear'} if opts.use_plugin is not None: from yaml import load as loadyml with open(opts.use_plugin) as pfile: plugin_settings = loadyml(pfile) else: # Setup multiprocessing if settings['n_procs'] == 0: settings['n_procs'] = cpu_count() if settings['ants_nthreads'] == 0: if settings['n_procs'] > 1: # always leave one extra thread for non ANTs work, # don't use more than 8 threads - the speed up is minimal settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8) else: settings['ants_nthreads'] = 1 if settings['n_procs'] > 1: plugin_settings['plugin'] = 'MultiProc' plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']} if opts.mem_gb: plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb MRIQC_LOG.info( 'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s', __version__, ', '.join(analysis_levels), opts.participant_label, settings) # Process data types modalities = opts.modalities dataset = collect_bids_data( settings['bids_dir'], modalities=modalities, participant_label=opts.participant_label, session=opts.session_id, run=opts.run_id, task=opts.task_id, ) # Set up participant level if 'participant' in analysis_levels: workflow = Workflow(name='workflow_enumerator') workflow.base_dir = settings['work_dir'] wf_list = [] for mod in modalities: if not dataset[mod]: MRIQC_LOG.warn('No %s scans were found in %s', mod, settings['bids_dir']) continue wf_list.append(build_workflow(dataset[mod], mod, settings=settings)) if wf_list: workflow.add_nodes(wf_list) if not opts.dry_run: if plugin_settings['plugin'] == 'MultiProc' and opts.profile: import logging from nipype.pipeline.plugins.callback_log import log_nodes_cb plugin_settings['plugin_args']['status_callback'] = log_nodes_cb callback_log_path = op.join(log_dir, 'run_stats.log') logger = logging.getLogger('callback') logger.setLevel(logging.DEBUG) handler = logging.FileHandler(callback_log_path) logger.addHandler(handler) workflow.run(**plugin_settings) if not settings['no_sub']: MRIQC_LOG.warn( 'Anonymized quality metrics have beeen submitted' ' to MRIQC\'s metrics repository.' ' Use --no-sub to disable submission.') if callback_log_path is not None: from nipype.utils.draw_gantt_chart import generate_gantt_chart generate_gantt_chart(callback_log_path, cores=settings['n_procs']) else: raise RuntimeError('Error reading BIDS directory (%s), or the dataset is not ' 'BIDS-compliant.' % settings['bids_dir']) # Set up group level if 'group' in analysis_levels: from mriqc.reports import group_html from mriqc.utils.misc import generate_csv, generate_pred reports_dir = check_folder(op.join(settings['output_dir'], 'reports')) derivatives_dir = op.join(settings['output_dir'], 'derivatives') n_group_reports = 0 for mod in modalities: dataframe, out_csv = generate_csv(derivatives_dir, settings['output_dir'], mod) # If there are no iqm.json files, nothing to do. if dataframe is None: MRIQC_LOG.warn( 'No IQM-JSON files were found for the %s data type in %s. The group-level ' 'report was not generated.', mod, derivatives_dir) continue MRIQC_LOG.info('Summary CSV table for the %s data generated (%s)', mod, out_csv) # out_pred = generate_pred(derivatives_dir, settings['output_dir'], mod) # if out_pred is not None: # MRIQC_LOG.info('Predicted QA CSV table for the %s data generated (%s)', # mod, out_pred) out_html = op.join(reports_dir, mod + '_group.html') group_html(out_csv, mod, csv_failed=op.join(settings['output_dir'], 'failed_' + mod + '.csv'), out_file=out_html) MRIQC_LOG.info('Group-%s report generated (%s)', mod, out_html) n_group_reports += 1 if n_group_reports == 0: raise Exception("No data found. No group level reports were generated.")
def main(): """Entry point""" from nipype import config as ncfg, logging as nlog from nipype.pipeline.engine import Workflow from .. import logging from ..utils.bids import collect_bids_data from ..workflows.core import build_workflow from ..utils.misc import check_folder # Run parser opts = get_parser().parse_args() # Retrieve logging level log_level = int(max(3 - opts.verbose_count, 0) * 10) if opts.verbose_count > 1: log_level = int(max(25 - 5 * opts.verbose_count, 1)) logging.getLogger().setLevel(log_level) log = logging.getLogger('mriqc.cli') # Build settings dict bids_dir = op.abspath(opts.bids_dir) # Number of processes n_procs = opts.n_procs settings = { 'bids_dir': bids_dir, 'write_graph': opts.write_graph, 'testing': opts.testing, 'hmc_afni': opts.hmc_afni, 'hmc_fsl': opts.hmc_fsl, 'fft_spikes_detector': opts.fft_spikes_detector, 'n_procs': n_procs, 'ants_nthreads': opts.ants_nthreads, 'ants_float': opts.ants_float, 'output_dir': op.abspath(opts.output_dir), 'work_dir': op.abspath(opts.work_dir), 'verbose_reports': opts.verbose_reports or opts.testing, 'float32': opts.float32, 'ica': opts.ica, 'no_sub': opts.no_sub, 'email': opts.email, 'fd_thres': opts.fd_thres, 'webapi_url': opts.webapi_url, 'webapi_port': opts.webapi_port, 'upload_strict': opts.upload_strict, } if opts.hmc_afni: settings['deoblique'] = opts.deoblique settings['despike'] = opts.despike settings['correct_slice_timing'] = opts.correct_slice_timing if opts.start_idx: settings['start_idx'] = opts.start_idx if opts.stop_idx: settings['stop_idx'] = opts.stop_idx if opts.ants_settings: settings['ants_settings'] = opts.ants_settings log_dir = op.join(settings['output_dir'], 'logs') analysis_levels = opts.analysis_level if opts.participant_label is None: analysis_levels.append('group') analysis_levels = list(set(analysis_levels)) if len(analysis_levels) > 2: raise RuntimeError('Error parsing analysis levels, got "%s"' % ', '.join(analysis_levels)) settings['report_dir'] = opts.report_dir if not settings['report_dir']: settings['report_dir'] = op.join(settings['output_dir'], 'reports') check_folder(settings['output_dir']) if 'participant' in analysis_levels: check_folder(settings['work_dir']) check_folder(log_dir) check_folder(settings['report_dir']) # Set nipype config ncfg.update_config({ 'logging': { 'log_directory': log_dir, 'log_to_file': True }, 'execution': { 'crashdump_dir': log_dir, 'crashfile_format': 'txt', 'resource_monitor': opts.profile }, }) # Set nipype logging level nlog.getLogger('workflow').setLevel(log_level) nlog.getLogger('interface').setLevel(log_level) nlog.getLogger('utils').setLevel(log_level) plugin_settings = {'plugin': 'Linear'} if opts.use_plugin is not None: from yaml import load as loadyml with open(opts.use_plugin) as pfile: plugin_settings = loadyml(pfile) else: # Setup multiprocessing if settings['n_procs'] == 0: settings['n_procs'] = cpu_count() if settings['ants_nthreads'] == 0: if settings['n_procs'] > 1: # always leave one extra thread for non ANTs work, # don't use more than 8 threads - the speed up is minimal settings['ants_nthreads'] = min(settings['n_procs'] - 1, 8) else: settings['ants_nthreads'] = 1 if settings['n_procs'] > 1: plugin_settings['plugin'] = 'MultiProc' plugin_settings['plugin_args'] = {'n_procs': settings['n_procs']} if opts.mem_gb: plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb # Process data types modalities = opts.modalities # Set up participant level if 'participant' in analysis_levels: log.info('Participant level started. Checking BIDS dataset...') dataset = collect_bids_data( settings['bids_dir'], modalities=modalities, participant_label=opts.participant_label, session=opts.session_id, run=opts.run_id, task=opts.task_id, ) log.info( 'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s', __version__, ', '.join(analysis_levels), opts.participant_label, settings) workflow = Workflow(name='workflow_enumerator') workflow.base_dir = settings['work_dir'] wf_list = [] for mod in modalities: if not dataset[mod]: log.warning('No %s scans were found in %s', mod, settings['bids_dir']) continue wf_list.append(build_workflow(dataset[mod], mod, settings=settings)) if wf_list: workflow.add_nodes(wf_list) if not opts.dry_run: # Warn about submitting measures BEFORE if not settings['no_sub']: log.warning('Anonymized quality metrics will be submitted' ' to MRIQC\'s metrics repository.' ' Use --no-sub to disable submission.') # run MRIQC workflow.run(**plugin_settings) # Warn about submitting measures AFTER if not settings['no_sub']: log.warning( 'Anonymized quality metrics have beeen submitted' ' to MRIQC\'s metrics repository.' ' Use --no-sub to disable submission.') else: msg = 'Error reading BIDS directory ({}), or the dataset is not ' \ 'BIDS-compliant.' if opts.participant_label or opts.session_id or opts.run_id or opts.task_id: msg = 'The combination of supplied labels' if opts.participant_label is not None: msg += ' (--participant_label {})'.format(" ".join( opts.participant_label)) if opts.session_id is not None: msg += ' (--session-id {})'.format(" ".join( opts.session_id)) if opts.run_id is not None: msg += ' (--run-id {})'.format(" ".join(opts.run_id)) if opts.task_id is not None: msg += ' (--task-id {})'.format(" ".join(opts.task_id)) msg += ' did not result in matches within the BIDS directory ({}).' raise RuntimeError(msg.format(settings['bids_dir'])) log.info('Participant level finished successfully.') # Set up group level if 'group' in analysis_levels: from ..reports import group_html from ..utils.misc import generate_csv # , generate_pred log.info('Group level started...') log.info( 'Running MRIQC-%s (analysis_levels=[%s], participant_label=%s)\n\tSettings=%s', __version__, ', '.join(analysis_levels), opts.participant_label, settings) reports_dir = check_folder(op.join(settings['output_dir'], 'reports')) derivatives_dir = op.join(settings['output_dir'], 'derivatives') n_group_reports = 0 for mod in modalities: dataframe, out_csv = generate_csv(derivatives_dir, settings['output_dir'], mod) # If there are no iqm.json files, nothing to do. if dataframe is None: log.warning( 'No IQM-JSON files were found for the %s data type in %s. The group-level ' 'report was not generated.', mod, derivatives_dir) continue log.info('Summary CSV table for the %s data generated (%s)', mod, out_csv) # out_pred = generate_pred(derivatives_dir, settings['output_dir'], mod) # if out_pred is not None: # log.info('Predicted QA CSV table for the %s data generated (%s)', # mod, out_pred) out_html = op.join(reports_dir, mod + '_group.html') group_html(out_csv, mod, csv_failed=op.join(settings['output_dir'], 'failed_' + mod + '.csv'), out_file=out_html) log.info('Group-%s report generated (%s)', mod, out_html) n_group_reports += 1 if n_group_reports == 0: raise Exception( "No data found. No group level reports were generated.") log.info('Group level finished successfully.')