def test_force_index(layout_ds005): data_dir = join(get_test_data_path(), 'ds005') target = join(data_dir, 'models', 'ds-005_type-test_model.json') indexer = BIDSLayoutIndexer(force_index=['models']) model_layout = BIDSLayout(data_dir, validate=True, indexer=indexer) assert target not in layout_ds005.files assert target in model_layout.files assert 'all' not in model_layout.get_subjects() for f in model_layout.files.values(): assert 'derivatives' not in f.path
def create_datasource_indiv_params(output_query, data_dir, indiv_params, subjects=None, sessions=None, acquisitions=None, reconstructions=None): """ Create a datasource node that have iterables following BIDS format, including a indiv_params file""" bids_datasource = pe.Node(interface=BIDSDataGrabberParams(indiv_params), name='bids_datasource') bids_datasource.inputs.base_dir = data_dir bids_datasource.inputs.output_query = output_query layout = BIDSLayout(data_dir) # Verbose print("BIDS layout:", layout) print("\t", layout.get_subjects()) print("\t", layout.get_sessions()) if subjects is None: subjects = layout.get_subjects() if sessions is None: sessions = layout.get_sessions() iterables = [] iterables.append(('subject', subjects)) iterables.append(('session', sessions)) if acquisitions is not None: iterables.append(('acquisition', acquisitions)) if reconstructions is not None: iterables.append(('reconstruction', reconstructions)) bids_datasource.iterables = iterables return bids_datasource
def create_datasource_noT1(data_dir, subjects=None, sessions=None, acquisitions=None): """ Create a datasource node that have iterables following BIDS format """ bids_datasource = pe.Node(interface=nio.BIDSDataGrabber(), name='bids_datasource') bids_datasource.inputs.base_dir = data_dir bids_datasource.inputs.output_query = { 'T1': { "datatype": "anat", "suffix": "T1w", "extensions": ["nii", ".nii.gz"] } } layout = BIDSLayout(data_dir) # Verbose print("BIDS layout:", layout) print("\t", layout.get_subjects()) print("\t", layout.get_sessions()) if subjects is None: subjects = layout.get_subjects() if sessions is None: sessions = layout.get_sessions() iterables = [] iterables.append(('subject', subjects)) iterables.append(('session', sessions)) if acquisitions is not None: iterables.append(('acquisition', acquisitions)) bids_datasource.iterables = iterables return bids_datasource
def get_files(): with open('dset_config.json', 'r') as fo: CONFIG = json.load(fo) DATA_DIR = op.abspath('/home/data/nbc/external-datasets/ds001491/') all_info = {} for dset_name in list(CONFIG.keys())[:3]: layout = BIDSLayout(op.join(DATA_DIR, dset_name)) cfg = CONFIG[dset_name] task = cfg['task'] dset_info = {} for sub in layout.get_subjects(): runs = layout.get_runs(subject=sub, task=task) sub_info = {} for run in runs: run_info = {} run_info['files'] = [] run_info['echo_times'] = [] for echo in sorted( layout.get_echoes(subject=sub, task=task, run=run)): raw_files = layout.get(subject=sub, task=task, run=run, echo=echo, extensions='.nii.gz') preproc_files = layout.get(subject=sub, task=task, run=run, root='afni-step1', echo=echo, extensions='.nii.gz', desc='realign') preproc_files = raw_files[:] if len(preproc_files) != 1: print(preproc_files) raise Exception('BAD') # Replace filename with path when using new version of bids run_info['files'].append(preproc_files[0].filename) metadata = layout.get_metadata(preproc_files[0].filename) run_info['echo_times'].append(metadata['EchoTime']) sub_info[run] = run_info dset_info[sub] = sub_info all_info[dset_name] = dset_info with open('all_files.json', 'w') as fo: json.dump(all_info, fo, indent=4, sort_keys=True)
def _run_interface(self, runtime): from bids.layout import BIDSLayout layout = BIDSLayout(self.inputs.bids_dir, derivatives=True) entities = [] extensions = ['preproc_bold.nii.gz'] for subject in np.sort(layout.get_subjects()): file = layout.get(subject=subject, task=layout.get_tasks(), extensions=extensions) if file == []: pass else: entity = {'subject': subject} #, 'session': session} entities.append(entity) self._results['entities'] = entities return runtime
def get_files(): DSET_DIR = op.abspath('/home/data/nbc/external-datasets/ds001491/') layout = BIDSLayout(DSET_DIR) layout.add_derivatives( '/home/data/nbc/external-datasets/ds001491/derivatives/afni-step1/') task = 'images' info = {} for sub in sorted(layout.get_subjects()): print(sub) sub_info = {'files': [], 'echo_times': []} for echo in sorted(layout.get_echoes(subject=sub, task=task)): raw_files = layout.get(subject=sub, task=task, echo=echo, extensions='.nii.gz') preproc_files = layout.get(subject=sub, task=task, root='afni-step1', extensions='.nii.gz') # For some reason pybids doesn't index echo in derivatives preproc_files = [ p for p in preproc_files if 'echo-{0}'.format(echo) in p.filename ] if len(preproc_files) != 1: print(preproc_files) raise Exception('BAD') # Replace filename with path when using new version of bids sub_info['files'].append(preproc_files[0].path) metadata = raw_files[0].metadata sub_info['echo_times'].append(metadata['EchoTime']) info[sub] = sub_info with open('preproc_files.json', 'w') as fo: json.dump(info, fo, indent=4, sort_keys=True)
print(line) if line == '' and process.poll() is not None: break if process.returncode != 0: raise Exception("Non zero return code: {0}\n" "{1}\n\n{2}".format(process.returncode, command, process.stdout.read())) if __name__ == '__main__': dset_dir = '/home/data/nbc/Sutherland_HIVCB/dset/' layout = BIDSLayout(dset_dir) log_files = glob('*.log') log_niis = [lf.replace('.log', '') for lf in log_files] cmd = ('/home/data/nbc/tools/deface/mri_deface {0} ' '/home/data/nbc/tools/deface/talairach_mixed_with_skull.gca ' '/home/data/nbc/tools/deface/face.gca {0}') subjects = layout.get_subjects() for subj in subjects: # Anatomical scans scans = layout.get(subject=subj, extensions='nii.gz', type='T1w') for scan in scans: f = scan.filename if not any([ln in f for ln in log_niis]): scan_cmd = cmd.format(f) run(scan_cmd)
def runpipeline(parser): # Parse inputs args = parser.parse_args() print(args) # 1) parse required inputs bids_dir = args.bids_dir participant = args.participant_label # 2) parse optional inputs nthreads = int(args.nthreads) bet_thr = float(args.bet_thr) small_fov = bool(args.small_fov) read_task_SNR = bool(args.taskSNR) # 2a) Need BIDS directory if no subjects chosen, use BIDSDataGrabber for this layout = BIDSLayout(bids_dir) if args.subjects: subject_id = [s for s in args.subjects.split(",")] print(subject_id) else: subject_id = layout.get_subjects() # 2b) set directories for results deriv_dir = op.join(op.realpath(bids_dir), "derivatives") # Set work & crash directories if args.work_dir: work_dir = op.realpath(args.work_dir) crash_dir = op.join(op.realpath(args.work_dir), "crash") else: work_dir = op.join(bids_dir, "derivatives/work") crash_dir = op.join(op.join(op.realpath(bids_dir), "derivatives/work"), "crash") if len(subject_id) == 1: work_dir = op.join(work_dir, subject_id[0]) crash_dir = op.join(work_dir, "crash") if not op.exists(work_dir): os.makedirs(work_dir) if not op.exists(crash_dir): os.makedirs(crash_dir) # 2c) set output directories if args.out_dir: out_dir = op.realpath(args.out_dir) else: out_dir = op.join(deriv_dir, "phaseprep") config.update_config({ "logging": { "log_directory": work_dir, "log_to_file": True, }, "execution": { "crashdump_dir": crash_dir, "crashfile_format": "txt", "hash_method": "content", "remove_unnecessary_outputs": False, }, }) logging.update_logging(config) phaseprep = pe.Workflow(name="phaseprep") phaseprep.base_dir = work_dir sink_dict = {} infosource = pe.Node(interface=ul.IdentityInterface(fields=["subject_id"]), name="infosource") infosource.iterables = [("subject_id", subject_id)] filegrabber = pe.Node( ul.Function( function=get_magandphase, input_names=["bids_dir", "subject_id"], output_names=["maglist", "phaselist"], ), name="filegrabber", ) filegrabber.inputs.bids_dir = bids_dir phaseprep.connect([(infosource, filegrabber, [("subject_id", "subject_id") ])]) # Step two will be magnitude preprocessing preproc_mag_wf = create_preprocess_mag_wf() preproc_mag_wf.inputs.inputspec.frac = bet_thr preproc_mag_wf.inputs.extractor.robust = small_fov sink_dict["procmag"] = pe.MapNode( DerivativesDataSink( desc="procmag", out_path_base="phaseprep", base_directory="out_dir", compress=True, ), name="dsink_procmag", iterfield=["in_file", "source_file"], ) phaseprep.connect([ ( filegrabber, preproc_mag_wf, [ ("maglist", "inputspec.input_mag"), (("maglist", get_tasklength), "inputspec.task"), (("maglist", get_restlength), "inputspec.rest"), ], ), ( preproc_mag_wf, sink_dict["procmag"], [("outputspec.proc_mag", "in_file")], ), (filegrabber, sink_dict["procmag"], [("maglist", "source_file")]), ]) # Step three will be phase preprocessing preproc_phase_wf = create_preprocess_phase_wf() sink_dict["procphase"] = sink_dict["procmag"].clone("procphase") sink_dict["procphase"].inputs.desc = "procphase" phaseprep.connect([ ( filegrabber, preproc_phase_wf, [ ("phaselist", "inputspec.input_phase"), ("maglist", "inputspec.input_mag"), (("phaselist", get_tasklength), "inputspec.task"), (("phaselist", get_restlength), "inputspec.rest"), ], ), ( preproc_mag_wf, preproc_phase_wf, [ ("outputspec.motion_par", "inputspec.motion_par"), ("outputspec.mask_file", "inputspec.mask_file"), ], ), ( preproc_phase_wf, sink_dict["procphase"], [("outputspec.proc_phase", "in_file")], ), (filegrabber, sink_dict["procphase"], [("phaselist", "source_file")]), ]) # Regress ge magnitude and phase phaseregress = pe.MapNode( interface=PhaseFitOdr.PhaseFitOdr(), name="phaseregressodr", iterfield=["phase", "mag", "TR"], ) phaseregress.iterables = ("noise_lb", [0.1, 0.15, 0.25, 0.4]) phaseregress.inputs.n_threads = 1 sink_dict["micro"] = sink_dict["procmag"].clone("micro") sink_dict["micro"].inputs.desc = "micro" sink_dict["macro"] = sink_dict["procmag"].clone("macro") sink_dict["macro"].inputs.desc = "macro" sink_dict["r2"] = sink_dict["procmag"].clone("r2") sink_dict["r2"].inputs.desc = "r2" sink_dict["beta"] = sink_dict["procmag"].clone("beta") sink_dict["beta"].inputs.desc = "beta" phaseprep.connect([ ( preproc_mag_wf, phaseregress, [ ("outputspec.proc_mag", "mag"), (("outputspec.proc_mag", get_TR), "TR"), ], ), (preproc_phase_wf, phaseregress, [("outputspec.proc_phase", "phase")]), (phaseregress, sink_dict["macro"], [("filt", "in_file")]), (filegrabber, sink_dict["macro"], [("maglist", "source_file")]), (phaseregress, sink_dict["micro"], [("sim", "in_file")]), (filegrabber, sink_dict["micro"], [("maglist", "source_file")]), (phaseregress, sink_dict["r2"], [("corr", "in_file")]), (filegrabber, sink_dict["r2"], [("maglist", "source_file")]), (phaseregress, sink_dict["beta"], [("beta", "in_file")]), (filegrabber, sink_dict["beta"], [("maglist", "source_file")]), ]) # if tcompcor is true run with tcompcor applied as well if args.tcompcor: tcompcor = pe.MapNode(interface=TCompCor(), name="tcompcor", iterfield=["realigned_file"]) stripheader_tcompcor = pe.MapNode( interface=ul.Function( function=stripheader, input_names=["filename"], output_names=["new_filename"], ), name="stripheader_tcompcor", iterfield=["filename"], ) phaseregress_multi = pe.MapNode( interface=PhaseFitOdr.PhaseFitOdr(), name="phaseregress_multi", iterfield=["phase", "mag", "global_regressors", "TR"], ) phaseregress_multi.inputs.n_threads = 1 phaseregress_multi.iterables = ("noise_lb", [0.1, 0.15, 0.25, 0.4]) sink_dict["micro_tcomp"] = sink_dict["procmag"].clone("micro_tcomp") sink_dict["micro_tcomp"].inputs.desc = "micro_tcomp" sink_dict["macro_tcomp"] = sink_dict["procmag"].clone("macro_tcomp") sink_dict["macro_tcomp"].inputs.desc = "macro_tcomp" sink_dict["r2_tcomp"] = sink_dict["procmag"].clone("r2_tcomp") sink_dict["r2_tcomp"].inputs.desc = "r2_tcomp" sink_dict["beta_tcomp"] = sink_dict["procmag"].clone("beta_tcomp") sink_dict["beta_tcomp"].inputs.desc = "beta_tcomp" phaseprep.connect([ ( preproc_mag_wf, tcompcor, [ ("outputspec.proc_mag", "realigned_file"), ("outputspec.mask_file", "mask_files"), ], ), (tcompcor, stripheader_tcompcor, [("components_file", "filename") ]), ( stripheader_tcompcor, phaseregress_multi, [("new_filename", "global_regressors")], ), ( preproc_mag_wf, phaseregress_multi, [ ("outputspec.proc_mag", "mag"), (("outputspec.proc_mag", get_TR), "TR"), ], ), ( preproc_phase_wf, phaseregress_multi, [("outputspec.proc_phase", "phase")], ), (phaseregress_multi, sink_dict["macro_tcomp"], [("filt", "in_file") ]), (filegrabber, sink_dict["macro_tcomp"], [("maglist", "source_file") ]), (phaseregress_multi, sink_dict["micro_tcomp"], [("sim", "in_file") ]), (filegrabber, sink_dict["micro_tcomp"], [("maglist", "source_file") ]), (phaseregress_multi, sink_dict["r2_tcomp"], [("corr", "in_file")]), (filegrabber, sink_dict["r2_tcomp"], [("maglist", "source_file")]), (phaseregress, sink_dict["beta_tcomp"], [("beta", "in_file")]), (filegrabber, sink_dict["beta_tcomp"], [("maglist", "source_file") ]), ]) # Step five will be ongoing during the previous steps ensuring correct sinking # Step six will be running this into a report print("setup pipline succesfully") if not args.test: print("running pipeline") starttime = time.time() phaseprep.write_graph(format="png") phaseprep.run(plugin="MultiProc", plugin_args={"n_procs": nthreads}) print("completed pipeline in ", time.time() - starttime, " seconds.")
def eval_multiclass_metrics_dataset(dataset_name, man_analysis_name="manual_segmentation", suffix="space-orig_desc-brain_dseg", output_dir="evaluation_results"): data_dir = os.path.join(data_path, dataset_name) layout = BIDSLayout(data_dir) # Verbose print("BIDS layout:", layout) subjects = layout.get_subjects() sessions = layout.get_sessions() print(subjects) print(sessions) res_eval_path = os.path.join(data_dir, "derivatives", output_dir) try: os.makedirs(res_eval_path) except OSError: print("res_eval_path {} already exists".format(res_eval_path)) results = [] for sub in subjects: if sub in ["032139", "032141", "032142", "032143"]: continue for ses in sessions: print("**** Running multiclass sub {} ses {} ****".format( sub, ses)) man_mask_file = os.path.join( data_dir, "derivatives", man_analysis_name, "sub-{}".format(sub), "ses-{}".format(ses), "anat", "sub-{}_ses-{}_{}.nii.gz".format(sub, ses, suffix)) assert os.path.exists( man_mask_file), "Error, could not find file {}".format( man_mask_file) for auto_analysis_name in auto_analysis_names: auto_mask_file = os.path.join( data_dir, "derivatives", auto_analysis_name, "sub-{}".format(sub), "ses-{}".format(ses), "anat", "sub-{}_ses-{}_{}.nii.gz".format(sub, ses, suffix)) assert os.path.exists( auto_mask_file), "Error, could not find file {}".format( auto_mask_file) eval_name = "manual-{}".format(auto_analysis_name) print("Comparing multiclass {} and {}".format( man_analysis_name, auto_analysis_name)) list_res = compute_all_multiclass_metrics( man_mask_file, auto_mask_file) list_res.insert(0, eval_name) list_res.insert(0, ses) list_res.insert(0, sub) results.append(list_res) #sub, ses, eval_name, VP, FP, VN, FN, kappa, dice, JC df = pd.DataFrame(results, columns=[ "Subject", "Session", "Evaluation", "ICC", "VP", "FP", "VN", "FN", "Kappa", "Dice", "Jaccard" ]) csv_name = "multiclass_" + dataset_name + "_eval_res.csv" df.to_csv(os.path.join(res_eval_path, csv_name)) return df
ValueError: BIDS root does not exist: /tmp/tmp9g7ddldw/bids-grabber/examples/examples/BIDS #problem with base_dir bg.inputs.base_dir = "/mnt/Filbey/Evan/MJXProcessing/examples/examples/BIDS" bg.inputs.subject = 'M7500516' #bg.inputs.output_query = {'T1w': dict(type='anat')} res = bg.run() res.outputs print("done") """ def printMe(paths): print("\n\nanalyzing " + str(paths) + "\n\n") analyzeANAT = Node(Function(function=printMe, input_names=["paths"], output_names=[]), name="analyzeANAT") bg_all = Node(BIDSDataGrabber(), name='bids-grabber') bg_all.inputs.base_dir = '/mnt/Filbey/Evan/MJXProcessing/examples/examples/BIDS' bg_all.inputs.output_query = {'ses': dict(type='session')} bg_all.iterables = ('subject', layout.get_subjects()[0]) wf = Workflow(name="bids_demo") wf.connect(bg_all, "session", analyzeANAT, "paths") wf.run()
class events_class(object): def __init__(self, base_dir, task_id, sub_ids=None, verbose=True): assert type(base_dir) == str, "base_dir should be type(str)" assert type(task_id) == str, "task_id should be type(str)" self.base_dir = base_dir self.layout = BIDSLayout(base_dir) self.task_id = task_id if sub_ids == None: self.sub_ids = self.layout.get_subjects(task=task_id) else: assert type(sub_ids) == list, "sub_ids should be type(list)" for i in sub_ids: assert type( i) == str, "elements in sub_ids should be type(str)" assert any(i == j for j in self.layout.get_subjects(task=task_id) ), f"cannot find sub_ids with {task} task data" if verbose: print(f'{len(self.sub_ids)} subjects in {self.task_id} task') def get_timing(self, trial_type_cols=[], trimTRby=10, software='AFNI', write=True): ''' Throws timing information for each subject into dictionary ''' assert (software == 'AFNI' or software == 'SPM'), "software should be AFNI or SPM" self.events_dict = {} self.events_dict[self.task_id] = {} for sub_identifier in self.sub_ids: self.events_dict[self.task_id][sub_identifier] = {} run_ids = self.layout.get_runs(subject=sub_identifier, task=self.task_id) for run_identifier in run_ids: __eventFile__ = self.layout.get(suffix='events', task=self.task_id, run=run_identifier, extension='tsv', return_type='file') # get df of events information __trialInfo__ = pd.read_table(__eventFile__[0]) if run_identifier == 1: onsets = {} durations = {} if trial_type_cols == []: #if no columns for trial_types specified for column in __trialInfo__.columns[2:]: __conditions__ = sorted( list(set(__trialInfo__[column]))) # extracting onset and duration information for task, subject, run, trial_type if run_identifier == 1: onsets[column] = {} durations[column] = {} for itrial in __conditions__: if run_identifier == 1: onsets[column][itrial] = {} durations[column][itrial] = {} try: onsets[column][itrial][run_identifier] = ( list(__trialInfo__[__trialInfo__[column] == itrial].onset - trimTRby) ) # subtracting trimTRby due to remove dummy scans durations[column][itrial][run_identifier] = ( list(__trialInfo__[__trialInfo__[column] == itrial].duration)) except KeyError: onsets[column][itrial] = {} durations[column][itrial] = {} onsets[column][itrial][run_identifier] = ( list(__trialInfo__[__trialInfo__[column] == itrial].onset - trimTRby) ) # subtracting trimTRby due to remove dummy scans durations[column][itrial][run_identifier] = ( list(__trialInfo__[__trialInfo__[column] == itrial].duration)) else: for t1 in trial_type_cols: assert any(t1 == t2 for t2 in __trialInfo__.columns ), f"{t1} info is not in *events.tsv" for column in trial_type_cols: __conditions__ = sorted( list(set(__trialInfo__[column]))) # extracting onset and duration information for task, subject, run, trial_type if run_identifier == 1: onsets[column] = {} durations[column] = {} for itrial in __conditions__: if run_identifier == 1: onsets[column][itrial] = {} durations[column][itrial] = {} try: onsets[column][itrial][run_identifier] = ( list(__trialInfo__[__trialInfo__[column] == itrial].onset - trimTRby) ) # subtracting trimTRby due to remove dummy scans durations[column][itrial][run_identifier] = ( list(__trialInfo__[__trialInfo__[column] == itrial].duration)) except KeyError: onsets[column][itrial] = {} durations[column][itrial] = {} onsets[column][itrial][run_identifier] = ( list(__trialInfo__[__trialInfo__[column] == itrial].onset - trimTRby) ) # subtracting trimTRby due to remove dummy scans durations[column][itrial][run_identifier] = ( list(__trialInfo__[__trialInfo__[column] == itrial].duration)) self.events_dict[self.task_id][sub_identifier]['onsets'] = onsets self.events_dict[ self.task_id][sub_identifier]['durations'] = durations if write == True: if trial_type_cols == []: #if no columns for trial_types specified for column in __trialInfo__.columns[2:]: if column == 'response_time': #cannot handle response_times yet continue writeToPath = os.path.join(self.base_dir, 'derivatives', 'timing', f'sub-{sub_identifier}', column) if not os.path.exists(writeToPath): os.makedirs(writeToPath) for (onset_key, onset_val), (dur_key, dur_val) in zip( onsets[column].items(), durations[column].items()): assert ( onset_key == dur_key ), "onset conditions and duration conditions do not match!" with open( f'{writeToPath}/sub-{sub_identifier}_task-{self.task_id}_timing-{onset_key}.txt', 'w') as filehandle: for (run_onsets, run_durs) in zip(onset_val.values(), dur_val.values()): filehandle.writelines( "%f:%f\t" % (onset_time, dur_time) for (onset_time, dur_time ) in zip(run_onsets, run_durs)) filehandle.write("\n") else: for column in trial_type_cols: if column == 'response_time': #cannot handle response_times yet continue writeToPath = os.path.join(self.base_dir, 'derivatives', 'timing', f'sub-{sub_identifier}', column) if not os.path.exists(writeToPath): os.makedirs(writeToPath) for (onset_key, onset_val), (dur_key, dur_val) in zip( onsets[column].items(), durations[column].items()): assert ( onset_key == dur_key ), "onset conditions and duration conditions do not match!" with open( f'{writeToPath}/sub-{sub_identifier}_task-{self.task_id}_timing-{onset_key}.txt', 'w') as filehandle: for (run_onsets, run_durs) in zip(onset_val.values(), dur_val.values()): filehandle.writelines( "%f:%f\t" % (onset_time, dur_time) for (onset_time, dur_time ) in zip(run_onsets, run_durs)) filehandle.write("\n")
def collect_participants(bids_dir, participant_label=None, strict=False): """ List the participants under the BIDS root and checks that participants designated with the participant_label argument exist in that folder. Returns the list of participants to be finally processed. Requesting all subjects in a BIDS directory root: >>> collect_participants('ds114') ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10'] Requesting two subjects, given their IDs: >>> collect_participants('ds114', participant_label=['02', '04']) ['02', '04'] Requesting two subjects, given their IDs (works with 'sub-' prefixes): >>> collect_participants('ds114', participant_label=['sub-02', 'sub-04']) ['02', '04'] Requesting two subjects, but one does not exist: >>> collect_participants('ds114', participant_label=['02', '14']) ['02'] >>> collect_participants('ds114', participant_label=['02', '14'], ... strict=True) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): fmriprep.utils.bids.BIDSError: ... """ layout = BIDSLayout(bids_dir) all_participants = layout.get_subjects() # Error: bids_dir does not contain subjects if not all_participants: raise BIDSError( 'Could not find participants. Please make sure the BIDS data ' 'structure is present and correct. Datasets can be validated online ' 'using the BIDS Validator (http://incf.github.io/bids-validator/).\n' 'If you are using Docker for Mac or Docker for Windows, you ' 'may need to adjust your "File sharing" preferences.', bids_dir) # No --participant-label was set, return all if not participant_label: return all_participants # Drop sub- prefixes participant_label = [ sub[4:] if sub.startswith('sub-') else sub for sub in participant_label ] found_label = layout.get_subjects(subject=participant_label) if not found_label: raise BIDSError( 'Could not find participants [{}]'.format( ', '.join(participant_label)), bids_dir) # Warn if some IDs were not found notfound_label = sorted(set(participant_label) - set(found_label)) if notfound_label: exc = BIDSError( 'Some participants were not found: {}'.format( ', '.join(notfound_label)), bids_dir) if strict: raise exc warnings.warn(exc.msg, BIDSWarning) return found_label
def eval_monoclass_metrics_dataset(dataset_name, man_analysis_name="manual_segmentation", suffix="space-orig_desc-brain_mask", output_dir="evaluation_results"): bids_dir = os.path.join(bids_path) # was very long layout = BIDSLayout(bids_dir) ### from https://github.com/bids-standard/pybids/pull/523 layout = BIDSLayout( bids_dir, database_file="bidsdb.sql", ) # Verbose print("BIDS layout:", layout) all_subjects = layout.get_subjects() all_sessions = layout.get_sessions() print(all_subjects) print(all_sessions) data_dir = os.path.join(data_path, dataset_name) res_eval_path = os.path.join(data_dir, "derivatives", output_dir) try: os.makedirs(res_eval_path) except OSError: print("res_eval_path {} already exists".format(res_eval_path)) results = [] for sub in subjects: assert sub in all_subjects, "Error, subject {} was not found in bids dir {}".format( sub, all_subjects) for ses in sessions: assert ses in all_sessions, "Error, session {} was not found in bids dir {}".format( ses, all_subjects) print("**** Running monoclass sub {} ses {} ****".format(sub, ses)) man_mask_file = os.path.join( data_dir, "derivatives", man_analysis_name, "sub-{}".format(sub), "ses-{}".format(ses), "anat", "sub-{}_ses-{}_{}.nii.gz".format(sub, ses, suffix)) assert os.path.exists( man_mask_file), "Error, could not find file {}".format( man_mask_file) for auto_analysis_name in auto_analysis_names: auto_mask_file = os.path.join( data_dir, "derivatives", auto_analysis_name, "sub-{}".format(sub), "ses-{}".format(ses), "anat", "sub-{}_ses-{}_{}.nii.gz".format(sub, ses, suffix)) assert os.path.exists( auto_mask_file), "Error, could not find file {}".format( auto_mask_file) eval_name = "manual-{}".format(auto_analysis_name) print("Comparing monoclass {} and {}".format( man_analysis_name, auto_analysis_name)) list_res = compute_all_monoclass_metrics( man_mask_file, auto_mask_file, pref=os.path.join(res_eval_path, sub + "_" + ses + "_" + eval_name + "_")) list_res.insert(0, eval_name) list_res.insert(0, ses) list_res.insert(0, sub) results.append(list_res) #sub, ses, eval_name, VP, FP, VN, FN, kappa, dice, JC, LCE, GCE df = pd.DataFrame(results, columns=[ "Subject", "Session", "Evaluation", "VP", "FP", "VN", "FN", "Kappa", "Dice", "Jaccard", "LCE", "GCE" ]) csv_name = "monoclass_" + dataset_name + "_eval_res.csv" df.to_csv(os.path.join(res_eval_path, csv_name)) return df
def get_input_subjects(bids_input_folder): """ """ layout = BIDSLayout(bids_input_folder) return layout.get_subjects()
# set paths for spm: spm.SPMCommand.set_mlab_paths(paths=path_spm, matlab_cmd=path_matlab) MatlabCommand.set_default_paths(path_spm) MatlabCommand.set_default_matlab_cmd(path_matlab) sub_list = ['sub-01'] elif 'linux' in sys.platform: # path_matlab = '/home/mpib/wittkuhn/spm12.simg eval \$SPMMCRCMD' # path_matlab = opj('/home', 'beegfs', 'wittkuhn', 'tools', 'spm', 'spm12.simg eval \$SPMMCRCMD') singularity_cmd = 'singularity run -B /home/mpib/wittkuhn -B /mnt/beegfs/home/wittkuhn /home/mpib/wittkuhn/highspeed/highspeed-glm/tools/spm/spm12.simg' singularity_spm = 'eval \$SPMMCRCMD' path_matlab = ' '.join([singularity_cmd, singularity_spm]) spm.SPMCommand.set_mlab_paths(matlab_cmd=path_matlab, use_mcr=True) # grab the list of subjects from the bids data set: layout = BIDSLayout(opj(path_root, 'bids')) # get all subject ids: sub_list = sorted(layout.get_subjects()) # create a template to add the "sub-" prefix to the ids sub_template = ['sub-'] * len(sub_list) # add the prefix to all ids: sub_list = ["%s%s" % t for t in zip(sub_template, sub_list)] # if user defined to run specific subject sub_list = sub_list[int(sys.argv[1]):int(sys.argv[2])] # print the SPM version: print('using SPM version %s' % spm.SPMCommand().version) # ====================================================================== # DEFINE PBS CLUSTER JOB TEMPLATE (NEEDED WHEN RUNNING ON THE CLUSTER): # ====================================================================== job_template = """ #PBS -l walltime=10:00:00 #PBS -j oe #PBS -o /home/mpib/wittkuhn/highspeed/highspeed-glm/logs/glm
def main(): parser = makeParser() results = parser.parse_args() if results.boutiques: createDescriptor(parser, results) return 0 verb = results.verbose fsldir = results.fsldir mni152 = op.join(fsldir, "data", "standard", "MNI152_T1_2mm_brain.nii.gz") mni152bn = op.basename(mni152).split(".")[0] outdir = results.output_dir partis = results.participant_label labels = results.parcellation if verb: print("BIDS Dir: {0}".format(results.bids_dir), flush=True) print("Output Dir: {0}".format(results.output_dir), flush=True) print("Analysis level: {0}".format(results.analysis_level), flush=True) # This preprocessing workflow is modified from the FSL recommendations here: # https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/FDT/UserGuide # Step 0, 1: Begin interrogation of BIDS dataset # Due to current super-linear slowdown in BIDS Layout, exclude all but # participant of interest. Explored in the following Github issue: # https://github.com/bids-standard/pybids/issues/285 if partis is not None: pattrn = 'sub-(?!{0})(.*)$'.format("|".join(partis)) else: pattrn = '' dset = BIDSLayout(results.bids_dir, exclude=pattrn) subjects = dset.get_subjects() if results.participant_label is not None: subjects = [pl for pl in results.participant_label if pl in subjects] assert (len(subjects) > 0) if verb: print("Participants: {0}".format(", ".join(s for s in subjects)), flush=True) sessions = dset.get_sessions() if results.session_label is not None: sessions = [sl for sl in results.session_label if sl in sessions] assert (len(sessions) > 0) if verb: print("Sessions: {0}".format(", ".join(s for s in sessions)), flush=True) # Step 0, 2: Prune dataset to subjects/sessions that have necessary files ftypes = [".nii", ".bval", ".bvec"] collections = [] for subj in subjects: for sess in sessions: tf_dwi = dset.get(subject=subj, session=sess, datatype="dwi", suffix="dwi", return_type="file") tf_anat = dset.get(subject=subj, session=sess, datatype="anat", suffix="T1w", return_type="file") if (all(any(ftype in fl for fl in tf_dwi) for ftype in ftypes) and any(ftypes[0] in fl for fl in tf_anat)): collections += [{ "subject": subj, "session": sess, "anat": [t for t in tf_anat if ftypes[0] in t][0], "bval": [t for t in tf_dwi if ftypes[1] in t][0], "bvec": [t for t in tf_dwi if ftypes[2] in t][0], "dwi": [t for t in tf_dwi if ftypes[0] in t][0] }] else: if verb: print("Skipping sub-{0}".format(subj) + " / ses-{0} due to missing data.".format(sess), flush=True) complete_collection = [] for col in collections: dwibn = op.basename(col["dwi"]).split('.')[0] anatbn = op.basename(col["anat"]).split('.')[0] subses = op.join('sub-{0}'.format(col['subject']), 'ses-{0}'.format(col['session'])) derivdir_d = op.join(outdir, subses, "dwi") derivdir_a = op.join(outdir, subses, "anat") execute("mkdir -p {0}".format(derivdir_d), verbose=verb, skipif=op.isdir(derivdir_d)) execute("mkdir -p {0}".format(derivdir_a), verbose=verb, skipif=op.isdir(derivdir_a)) # Step 1: Extract B0 volumes # Make even number of spatial voxels? (req'd for eddy for some reason) # TODO : above, if actually needed - docs inconsistent # Get B0 locations with open(col["bval"]) as fhandle: bvals = fhandle.read().split(" ") bvals = [int(b) for b in bvals if b != '' and b != '\n'] b0_loc = [i for i, b in enumerate(bvals) if b == np.min(bvals)] # Get B0 volumes col["b0_scans"] = [] for idx, b0 in enumerate(b0_loc): b0ind = "b0_{0}".format(idx) col["b0_scans"] += [ op.join(derivdir_d, dwibn + "_" + b0ind + ".nii.gz") ] execute(fsl.fslroi(col["dwi"], col["b0_scans"][-1], *[b0, 1]), verbose=verb, skipif=op.isfile(col["b0_scans"][-1])) # Merge B0 volumes col["b0s"] = op.join(derivdir_d, dwibn + "_b0s.nii.gz") execute(fsl.fslmerge(col["b0s"], *col["b0_scans"]), verbose=verb, skipif=op.isfile(col["b0s"])) # Create acquisition parameters file col["acqparams"] = op.join(derivdir_d, dwibn + "_acq.txt") acqs = { "i": "1 0 0", "i-": "-1 0 0", "j": "0 1 0", "j-": "0 -1 0", "k": "0 0 1", "k-": "0 0 -1" } with open(col["acqparams"], 'w') as fhandle: meta = dset.get_metadata(path=col["dwi"]) pedir = meta["PhaseEncodingDirection"] trout = meta["TotalReadoutTime"] line = "{0} {1}".format(acqs[pedir], trout) fhandle.write("\n".join([line] * len(b0_loc))) # Step 1.5: Run Top-up on Diffusion data # TODO: remove; topup only applies with multiple PEs (rare in open data) # col["topup"] = op.join(derivdir_d, dwibn + "_topup") # col["hifi_b0"] = op.join(derivdir_d, dwibn + "_hifi_b0") # execute(fsl.topup(col["b0s"], col["acqparams"], # col["topup"], col["hifi_b0"]), # verbose=verb) # execute(fsl.fslmaths(col["hifi_b0"], "-Tmean", col["hifi_b0"]), # verbose=verb) # Step 2: Brain extraction # ... Diffusion: col["dwi_brain"] = op.join(derivdir_d, dwibn + "_brain.nii.gz") col["dwi_mask"] = op.join(derivdir_d, dwibn + "_brain_mask.nii.gz") execute(fsl.bet(col["dwi"], col["dwi_brain"], "-F", "-m"), verbose=verb, skipif=op.isfile(col["dwi_brain"])) # ... Structural: col["anat_brain"] = op.join(derivdir_a, anatbn + "_brain.nii.gz") col["anat_mask"] = op.join(derivdir_a, anatbn + "_brain.nii.gz") execute(fsl.bet(col["anat"], col["anat_brain"], "-m"), verbose=verb, skipif=op.isfile(col["anat_brain"])) # Step 3: Produce prelimary DTIfit QC figures col["dwi_qc_pre"] = op.join(derivdir_d, dwibn + "_dtifit_pre") execute(fsl.dtifit(col["dwi_brain"], col["dwi_qc_pre"], col["dwi_mask"], col["bvec"], col["bval"]), verbose=verb, skipif=op.isfile(col["dwi_qc_pre"] + "_FA.nii.gz")) # Step 4: Perform eddy correction # ... Create index col["index"] = op.join(derivdir_d, dwibn + "_eddy_index.txt") with open(col["index"], 'w') as fhandle: fhandle.write(" ".join(["1"] * len(bvals))) # ... Run eddy col["eddy_dwi"] = op.join(derivdir_d, dwibn + "_eddy") if results.gpu: eddy_exe = "eddy_cuda8.0" else: eddy_exe = "eddy_openmp" execute(fsl.eddy(col["dwi_brain"], col["dwi_mask"], col["acqparams"], col["index"], col["bvec"], col["bval"], col["eddy_dwi"], exe=eddy_exe), verbose=verb, skipif=op.isfile(col["eddy_dwi"] + ".nii.gz")) # Step 5: Registration to template # ... Compute transforms col["t1w2mni"] = op.join(derivdir_a, anatbn + "_to_mni_xfm.mat") execute(fsl.flirt(col["anat_brain"], omat=col["t1w2mni"], ref=mni152), verbose=verb, skipif=op.isfile(col["t1w2mni"])) col["dwi2t1w"] = op.join(derivdir_d, dwibn + "_to_t1w_xfm.mat") execute(fsl.flirt(col["eddy_dwi"], ref=col["anat_brain"], omat=col["dwi2t1w"]), verbose=verb, skipif=op.isfile(col["dwi2t1w"])) col["dwi2mni"] = op.join(derivdir_d, dwibn + "_to_mni_xfm.mat") execute(fsl.convert_xfm(concat=col["t1w2mni"], inp=col["dwi2t1w"], omat=col["dwi2mni"]), verbose=verb, skipif=op.isfile(col["dwi2mni"])) # ... Invert transforms towards diffusion space col["mni2dwi"] = op.join(derivdir_d, dwibn + "_from_mni_xfm.mat") execute(fsl.convert_xfm(inverse=col["dwi2mni"], omat=col["mni2dwi"]), verbose=verb, skipif=op.isfile(col["mni2dwi"])) col["t1w2dwi"] = op.join(derivdir_a, anatbn + "_dwi_xfm.mat") execute(fsl.convert_xfm(inverse=col["dwi2t1w"], omat=col["t1w2dwi"]), verbose=verb, skipif=op.isfile(col["t1w2dwi"])) # Step 6: Apply registrations to anatomical and template images col["anat_in_dwi"] = op.join(derivdir_a, anatbn + "_brain_dwi.nii.gz") execute(fsl.flirt(col["anat_brain"], applyxfm=True, out=col["anat_in_dwi"], init=col["t1w2dwi"], ref=col["eddy_dwi"]), verbose=verb, skipif=op.isfile(col["anat_in_dwi"])) col["mni_in_dwi"] = op.join( derivdir_d, ("atlas_" + dwibn + "_" + mni152bn + "_dwi.nii.gz")) execute(fsl.flirt(mni152, applyxfm=True, out=col["mni_in_dwi"], init=col["mni2dwi"], ref=col["eddy_dwi"]), verbose=verb, skipif=op.isfile(col["mni_in_dwi"])) # Step 7: Perform tissue segmentation on anatomical images in DWI space col["tissue_masks"] = op.join(derivdir_d, anatbn + "_fast") execute(fsl.fast(col["anat_in_dwi"], col["tissue_masks"], classes=3, imtype=1), verbose=verb, skipif=op.isfile(col["tissue_masks"] + "_seg_2.nii.gz")) # Step 8: Transform parcellations into DWI space col["labels_in_dwi"] = [] for label in labels: lbn = op.basename(label).split('.')[0] col["labels_in_dwi"] += [ op.join(derivdir_d, ("labels_" + dwibn + "_" + lbn + ".nii.gz")) ] execute(fsl.flirt(label, applyxfm=True, out=col["labels_in_dwi"][-1], init=col["mni2dwi"], ref=col["eddy_dwi"], interp="nearestneighbour"), verbose=verb, skipif=op.isfile(col["labels_in_dwi"][-1])) if verb: print("Finished processing sub-{0}".format(subj) + " / ses-{0} !".format(sess), flush=True) complete_collection += [col]
def makeInput(self, inp): """ Obtains the input (file-paths and input-mode) from the main window and updates the subjects/time-series objects appropriately """ # obtains the Raw BOLD data and preprocessed data, and updates the data-store # each input comprises of an input_file, mask_file and a file_type, and the mode input_file, mask_file, file_type, mode = inp # if the mode is file (that is input is a stand-alone NIfTI or GIfTI file) if mode == "file": return self.makeInputFile(input_file, mask_file, file_type, 'file') # if the input is present in a directory organized in BIDS format elif "bids" in mode: # obtaining all the input files layout = BIDSLayout(input_file, validate=False, config=['bids', 'derivatives']) subjects_to_analyze = layout.get_subjects() # if no subjects were found if not subjects_to_analyze: return Status( False, error= 'Could not find participants. Please make sure the BIDS data ' 'structure is present and correct. Datasets can be validated online ' 'using the BIDS Validator (http://incf.github.io/bids-validator/).' ) all_inputs = layout.get(return_type='filename', datatype='func', subject=subjects_to_analyze, task='rest', desc='preproc', suffix='bold', extension=['nii', 'nii.gz']) # if no *bold.nii or *bold.nii.gz files were found if not all_inputs != []: return Status( False, error='There are no files of type *bold.nii / *bold.nii.gz ' 'Please make sure to have at least one file of the above type ' 'in the BIDS specification') # if the atlas file is provided if mode == "bids w/ atlas": for file_count in range(len(all_inputs)): # making input for every combination of input-file and atlas temp = (self.makeInputFile(all_inputs[file_count], mask_file, ".nii", 'bids w/ atlas')) if not temp.get_state(): return temp # returns the status return Status( True, info="Preprocessed all input", dic={"Number of Input Subjects: ": len(all_inputs)}) # if the atlas file is not provided elif mode == "bids": # obtaining the masks all_masks = layout.get(return_type='filename', datatype='func', subject=subjects_to_analyze, task='rest', desc='brain', suffix='mask', extension=['nii', 'nii.gz']) # if no mask files were found if not all_masks != []: return Status( False, error= 'There are no files of type *mask.nii / *mask.nii.gz ' 'Please make sure to have at least one file of the above type ' 'in the BIDS specification') # if the number of mask files != number of input files if len(all_inputs) != len(all_masks): return Status( False, error= 'The number of *bold.nii / .nii.gz and the number of ' '*mask.nii / .nii.gz are different. Please make sure that ' 'there is one mask for each input_file present') all_inputs.sort() all_masks.sort() # matching the prefix to align the inputs and the masks together all_prefix_match = False prefix_match_count = 0 for i in range(len(all_inputs)): input_prefix = all_inputs[i].split('/')[-1].split( '_desc')[0] mask_prefix = all_masks[i].split('/')[-1].split('_desc')[0] if input_prefix == mask_prefix: prefix_match_count += 1 else: all_prefix_match = False break if prefix_match_count == len(all_inputs): all_prefix_match = True # if the mask and input files have different prefix if not all_prefix_match: return Status( False, error= 'The mask and input files should have the same prefix for correspondence. ' 'Please consider renaming your files') else: # making input for every combination of input-file and mask-file for file_count in range(len(all_inputs)): temp = (self.makeInputFile(all_inputs[file_count], all_masks[file_count], ".nii", 'bids')) if not temp.get_state(): return temp # returns the status return Status( True, info="Preprocessed all input", dic={"Number of Input Subjects: ": len(all_inputs)})
fig.tight_layout() fig.savefig(op.join(out_dir, 'scrubbing_analysis.png'), dpi=400) del mean_delta_r, scrub_smc, perm_scrub_smc if __name__ == '__main__': """ Run a test using 40 subjects from the ADHD dataset. """ dset_dir = '/home/data/nbc/Sutherland_ACE/dset/' deriv_dir = '/home/data/nbc/Sutherland_ACE/derivatives/fmriprep-1.2.1/' fd_thresh = 0.2 n_iters = 10000 layout = BIDSLayout(dset_dir) subjects = layout.get_subjects(task='rest') motpars = [] files = [] for sub in subjects: preproc_file = op.join( deriv_dir, 'sub-{0}/ses-S1/func/sub-{0}_ses-S1_task-rest_run-01_space-' 'MNI152NLin2009cAsym_desc-preproc_bold.nii.gz'.format(sub)) files.append(preproc_file) conf_file = op.join( deriv_dir, 'sub-{0}/ses-S1/func/sub-{0}_ses-S1_task-rest_run-01_desc-' 'confounds_regressors.tsv'.format(sub)) conf = pd.read_csv(conf_file, sep='\t') motpars_ = conf[[ 'trans_x', 'trans_y', 'trans_z', 'rot_x', 'rot_y', 'rot_z'
def run_rsHRF(): parser = get_parser() args = parser.parse_args() arg_groups = {} for group in parser._action_groups: group_dict = { a.dest: getattr(args, a.dest, None) for a in group._group_actions } arg_groups[group.title] = group_dict para = arg_groups['Parameters'] nargs = len(sys.argv) temporal_mask = [] if (not args.GUI) and (args.output_dir is None): parser.error( '--output_dir is required when executing in command-line interface' ) if (not args.GUI) and (args.estimation is None): parser.error( '--estimation rule is required when executing in command-line interface' ) if (args.GUI): if (nargs == 2): try: from .rsHRF_GUI import run run.run() except ModuleNotFoundError: parser.error( '--GUI should not be used inside a Docker container') else: parser.error('--no other arguments should be supplied with --GUI') if (args.input_file is not None or args.ts is not None) and args.analysis_level: parser.error( 'analysis_level cannot be used with --input_file or --ts, do not supply it' ) if (args.input_file is not None or args.ts is not None) and args.participant_label: parser.error( 'participant_labels are not to be used with --input_file or --ts, do not supply it' ) if args.input_file is not None and args.brainmask: parser.error( '--brainmask cannot be used with --input_file, use --atlas instead' ) if args.ts is not None and (args.brainmask or args.atlas): parser.error( '--atlas or --brainmask cannot be used with --ts, do not supply it' ) if args.bids_dir is not None and not (args.brainmask or args.atlas): parser.error( '--atlas or --brainmask needs to be supplied with --bids_dir') if args.bids_dir is not None and not args.analysis_level: parser.error( 'analysis_level needs to be supplied with bids_dir, choices=[participant]' ) if args.input_file is not None and (not args.input_file.endswith( ('.nii', '.nii.gz', '.gii', '.gii.gz'))): parser.error( '--input_file should end with .gii, .gii.gz, .nii or .nii.gz') if args.atlas is not None and (not args.atlas.endswith( ('.nii', '.nii.gz', '.gii', '.gii.gz'))): parser.error('--atlas should end with .gii, .gii.gz, .nii or .nii.gz') if args.ts is not None and (not args.ts.endswith(('.txt'))): parser.error('--ts file should end with .txt') if args.temporal_mask is not None and (not args.temporal_mask.endswith( ('.dat'))): parser.error('--temporal_mask ile should end with ".dat"') if args.temporal_mask is not None: f = open(args.temporal_mask, 'r') for line in f: for each in line: if each in ['0', '1']: temporal_mask.append(int(each)) if args.estimation == 'sFIR' or args.estimation == 'FIR': para['T'] = 1 if args.ts is not None: file_type = op.splitext(args.ts) if para['TR'] <= 0: parser.error('Please supply a valid TR using -TR argument') else: TR = para['TR'] para['dt'] = para['TR'] / para['T'] para['lag'] = np.arange(np.fix(para['min_onset_search'] / para['dt']), np.fix(para['max_onset_search'] / para['dt']) + 1, dtype='int') fourD_rsHRF.demo_rsHRF(args.ts, None, args.output_dir, para, args.n_jobs, file_type, mode='time-series', temporal_mask=temporal_mask, wiener=args.wiener) if args.input_file is not None: if args.atlas is not None: if (args.input_file.endswith( ('.nii', '.nii.gz')) and args.atlas.endswith( ('.gii', '.gii.gz'))) or (args.input_file.endswith( ('.gii', '.gii.gz')) and args.atlas.endswith( ('.nii', '.nii.gz'))): parser.error( '--atlas and input_file should be of the same type [NIfTI or GIfTI]' ) # carry analysis with input_file and atlas file_type = op.splitext(args.input_file) if file_type[-1] == ".gz": file_type = op.splitext(file_type[-2])[-1] + file_type[-1] else: file_type = file_type[-1] if ".nii" in file_type: TR = (spm_dep.spm.spm_vol(args.input_file).header.get_zooms())[-1] else: if para['TR'] == -1: parser.error('Please supply a valid TR using -TR argument') else: TR = para['TR'] if TR <= 0: if para['TR'] <= 0: parser.error('Please supply a valid TR using -TR argument') else: if para['TR'] == -1: para['TR'] = TR elif para['TR'] <= 0: print('Invalid TR supplied, using implicit TR: {0}'.format(TR)) para['TR'] = TR para['dt'] = para['TR'] / para['T'] para['lag'] = np.arange(np.fix(para['min_onset_search'] / para['dt']), np.fix(para['max_onset_search'] / para['dt']) + 1, dtype='int') fourD_rsHRF.demo_rsHRF(args.input_file, args.atlas, args.output_dir, para, args.n_jobs, file_type, mode='input', temporal_mask=temporal_mask, wiener=args.wiener) if args.bids_dir is not None: utils.bids.write_derivative_description(args.bids_dir, args.output_dir) bids_dir = Path(args.bids_dir) fname = bids_dir / 'dataset_description.json' if fname.exists(): desc = json.loads(Path(fname).read_text()) if 'DataType' in desc: if desc['DataType'] != 'derivative': parser.error( 'Input data is not a derivative dataset' ' (DataType in dataset_description.json is not equal to "derivative")' ) else: parser.error( 'DataType is not defined in the dataset_description.json file. Please make sure DataType is defined. ' 'Information on the dataset_description.json file can be found online ' '(https://bids-specification.readthedocs.io/en/stable/03-modality-agnostic-files.html' '#derived-dataset-and-pipeline-description)') else: parser.error( 'Could not find dataset_description.json file. Please make sure the BIDS data ' 'structure is present and correct. Datasets can be validated online ' 'using the BIDS Validator (http://incf.github.io/bids-validator/).' ) if args.bids_dir is not None and args.atlas is not None: # carry analysis with bids_dir and 1 atlas layout = BIDSLayout(args.bids_dir, validate=False, config=['bids', 'derivatives']) if args.participant_label: input_subjects = args.participant_label subjects_to_analyze = layout.get_subjects(subject=input_subjects) else: subjects_to_analyze = layout.get_subjects() if not subjects_to_analyze: parser.error( 'Could not find participants. Please make sure the BIDS data ' 'structure is present and correct. Datasets can be validated online ' 'using the BIDS Validator (http://incf.github.io/bids-validator/).' ) if not args.atlas.endswith(('.nii', '.nii.gz')): parser.error('--atlas should end with .nii or .nii.gz') if args.bids_filter_file is not None: filter_list = json.loads(Path(args.bids_filter_file).read_text()) default_input = { 'extension': 'nii.gz', 'datatype': 'func', 'desc': 'preproc', 'task': 'rest', 'suffix': 'bold' } default_input['subject'] = subjects_to_analyze default_input.update(filter_list['bold']) all_inputs = layout.get(return_type='filename', **default_input) else: all_inputs = layout.get(return_type='filename', datatype='func', subject=subjects_to_analyze, task='rest', desc='preproc', suffix='bold', extension=['nii', 'nii.gz']) if not all_inputs != []: parser.error( 'There are no files of type *bold.nii / *bold.nii.gz ' 'Please make sure to have at least one file of the above type ' 'in the BIDS specification') else: num_errors = 0 for file_count in range(len(all_inputs)): try: TR = layout.get_metadata( all_inputs[file_count])['RepetitionTime'] except KeyError as e: TR = spm_dep.spm.spm_vol( all_inputs[file_count]).header.get_zooms()[-1] para['TR'] = TR para['dt'] = para['TR'] / para['T'] para['lag'] = np.arange( np.fix(para['min_onset_search'] / para['dt']), np.fix(para['max_onset_search'] / para['dt']) + 1, dtype='int') num_errors += 1 try: fourD_rsHRF.demo_rsHRF(all_inputs[file_count], args.atlas, args.output_dir, para, args.n_jobs, file_type, mode='bids w/ atlas', temporal_mask=temporal_mask, wiener=args.wiener) num_errors -= 1 except ValueError as err: print(err.args[0]) except: print("Unexpected error:", sys.exc_info()[0]) success = len(all_inputs) - num_errors if success == 0: raise RuntimeError( 'Dimensions were inconsistent for all input-mask pairs; \n' 'No inputs were processed!') if args.bids_dir is not None and args.brainmask: # carry analysis with bids_dir and brainmask layout = BIDSLayout(args.bids_dir, validate=False, config=['bids', 'derivatives']) if args.participant_label: input_subjects = args.participant_label subjects_to_analyze = layout.get_subjects(subject=input_subjects) else: subjects_to_analyze = layout.get_subjects() if not subjects_to_analyze: parser.error( 'Could not find participants. Please make sure the BIDS data ' 'structure is present and correct. Datasets can be validated online ' 'using the BIDS Validator (http://incf.github.io/bids-validator/).' ) if args.bids_filter_file is not None: filter_list = json.loads(Path(args.bids_filter_file).read_text()) default_input = { 'extension': 'nii.gz', 'datatype': 'func', 'desc': 'preproc', 'task': 'rest', 'suffix': 'bold' } default_input['subject'] = subjects_to_analyze default_input.update(filter_list['bold']) all_inputs = layout.get(return_type='filename', **default_input) default_mask = { 'extension': 'nii.gz', 'datatype': 'func', 'desc': 'brain', 'task': 'rest', 'suffix': 'mask' } default_mask['subject'] = subjects_to_analyze default_mask.update(filter_list['mask']) all_masks = layout.get(return_type='filename', **default_mask) else: all_inputs = layout.get(return_type='filename', datatype='func', subject=subjects_to_analyze, task='rest', desc='preproc', suffix='bold', extension=['nii', 'nii.gz']) all_masks = layout.get(return_type='filename', datatype='func', subject=subjects_to_analyze, task='rest', desc='brain', suffix='mask', extension=['nii', 'nii.gz']) if not all_inputs != []: parser.error( 'There are no files of type *bold.nii / *bold.nii.gz ' 'Please make sure to have at least one file of the above type ' 'in the BIDS specification') if not all_masks != []: parser.error( 'There are no files of type *mask.nii / *mask.nii.gz ' 'Please make sure to have at least one file of the above type ' 'in the BIDS specification') if len(all_inputs) != len(all_masks): parser.error( 'The number of *bold.nii / .nii.gz and the number of ' '*mask.nii / .nii.gz are different. Please make sure that ' 'there is one mask for each input_file present') all_inputs.sort() all_masks.sort() all_prefix_match = False prefix_match_count = 0 for i in range(len(all_inputs)): input_prefix = all_inputs[i].split('/')[-1].split('_desc')[0] mask_prefix = all_masks[i].split('/')[-1].split('_desc')[0] if input_prefix == mask_prefix: prefix_match_count += 1 else: all_prefix_match = False break if prefix_match_count == len(all_inputs): all_prefix_match = True if not all_prefix_match: parser.error( 'The mask and input files should have the same prefix for correspondence. ' 'Please consider renaming your files') else: num_errors = 0 for file_count in range(len(all_inputs)): file_type = all_inputs[file_count].split('bold')[1] if file_type == ".nii" or file_type == ".nii.gz": try: TR = layout.get_metadata( all_inputs[file_count])['RepetitionTime'] except KeyError as e: TR = spm_dep.spm.spm_vol( all_inputs[file_count]).header.get_zooms()[-1] para['TR'] = TR else: spm_dep.spm.spm_vol(all_inputs[file_count]) TR = spm_dep.spm.spm_vol( all_inputs[file_count]).get_arrays_from_intent( "NIFTI_INTENT_TIME_SERIES")[0].meta.get_metadata( )["TimeStep"] para['TR'] = float(TR) * 0.001 para['dt'] = para['TR'] / para['T'] para['lag'] = np.arange( np.fix(para['min_onset_search'] / para['dt']), np.fix(para['max_onset_search'] / para['dt']) + 1, dtype='int') num_errors += 1 try: fourD_rsHRF.demo_rsHRF(all_inputs[file_count], all_masks[file_count], args.output_dir, para, args.n_jobs, mode='bids', temporal_mask=temporal_mask, wiener=args.wiener) num_errors -= 1 except ValueError as err: print(err.args[0]) except: print("Unexpected error:", sys.exc_info()[0]) success = len(all_inputs) - num_errors if success == 0: raise RuntimeError( 'Dimensions were inconsistent for all input-mask pairs; \n' 'No inputs were processed!')
def read_bids_dataset(bids_input, subject_list=None, session_list=None, collect_on_subject=False): """ extracts and organizes relevant metadata from a bids dataset necessary for the dcan-modified hcp fmri processing pipeline. :param bids_input: path to input bids folder :param subject_list: EITHER, a list of subject ids to filter on, OR a dictionary of subject id: list of sessions to filter on. :param session_list: a list of session ids to filter on. :param collect_on_subject: collapses all sessions, for cases with non-longitudinal data spread across scan sessions. :return: bids data struct (nested dict) spec: { t1w: t1w filename list, t2w: t2w filename list, t1w_metadata: bids meta data (first t1), t2w_metadata: bids meta data (first t2), func: fmri filename list, func_metadata: bids meta data list, fmap: { positive: spin echo filename list (if applicable) negative: spin echo filename list (if applicable) }, fmap_metadata: { positive: bids meta data list (if applicable) negative: bids meta data list (if applicable) }, } """ layout = BIDSLayout(bids_input, index_metadata=True) subjects = layout.get_subjects() # filter subject list if isinstance(subject_list, list): subjects = [s for s in subjects if s in subject_list] elif isinstance(subject_list, dict): subjects = [s for s in subjects if s in subject_list.keys()] subsess = [] # filter session list for s in subjects: sessions = layout.get_sessions(subject=s) # filter sessions_list if isinstance(session_list, list): sessions = [t for t in sessions if t in session_list] if not sessions: subsess += [(s, None)] elif collect_on_subject: subsess += [(s, sessions)] else: subsess += list(product([s], sessions)) assert len(subsess), 'bids data not found for participants. If labels ' \ 'were provided, check the participant labels for errors. ' \ 'Otherwise check that the bids folder provided is correct.' for subject, sessions in subsess: # get relevant image datatypes anat, anat_types = set_anatomicals(layout, subject, sessions) func, func_types = set_functionals(layout, subject, sessions) fmap, fmap_types = set_fieldmaps(layout, subject, sessions) bids_data = { 'subject': subject, 'session': sessions if not collect_on_subject else None, 'types': anat_types.union(func_types, fmap_types) } bids_data.update(anat) bids_data.update(func) bids_data.update(fmap) yield bids_data
def find_data(self, subject_ids=None, visit_ids=None): """ Return subject and session information for a project in the local repository Parameters ---------- subject_ids : list(str) List of subject IDs with which to filter the tree with. If None all are returned visit_ids : list(str) List of visit IDs with which to filter the tree with. If None all are returned Returns ------- project : arcana.repository.Tree A hierarchical tree of subject, session and fileset information for the repository """ filesets = [] layout = BIDSLayout(self.root_dir) all_subjects = layout.get_subjects() all_visits = layout.get_sessions() if not all_visits: all_visits = [self.DEFAULT_VISIT_ID] self._depth = 1 else: self._depth = 2 for item in layout.get(return_type='object'): if item.path.startswith(self.derivatives_dir): # We handle derivatives using the BasicRepo base # class methods continue if not hasattr(item, 'entities') or not item.entities.get('suffix', False): logger.warning("Skipping unrecognised file '{}' in BIDS tree" .format(op.join(item.dirname, item.filename))) continue # Ignore hidden file try: subject_ids = [item.entities['subject']] except KeyError: # If item exists in top-levels of in the directory structure # it is inferred to exist for all subjects in the tree subject_ids = all_subjects try: visit_ids = [item.entities['session']] except KeyError: # If item exists in top-levels of in the directory structure # it is inferred to exist for all visits in the tree visit_ids = all_visits for subject_id in subject_ids: for visit_id in visit_ids: aux_files = {} metadata = layout.get_metadata(item.path) if metadata and not item.path.endswith('.json'): # Write out the combined JSON side cars to a temporary # file to include in extended NIfTI filesets metadata_path = op.join( self.metadata_dir, 'sub-{}'.format(subject_id), 'ses-{}'.format(visit_id), item.filename + '.json') os.makedirs(op.dirname(metadata_path), exist_ok=True) if not op.exists(metadata_path): with open(metadata_path, 'w') as f: json.dump(metadata, f) aux_files['json'] = metadata_path fileset = BidsFileset( path=op.join(item.dirname, item.filename), type=item.entities['suffix'], subject_id=subject_id, visit_id=visit_id, repository=self, modality=item.entities.get('modality', None), task=item.entities.get('task', None), aux_files=aux_files) filesets.append(fileset) # Get derived filesets, fields and records using the same method using # the method in the BasicRepo base class derived_filesets, fields, records = super().find_data( subject_ids=subject_ids, visit_ids=visit_ids) filesets.extend(derived_filesets) return filesets, fields, records
import os from bids.layout import BIDSLayout # Directory where your data set resides. dataDir = '/tmp/Data/ds114' # Creating the layout object for this BIDS data set layout = BIDSLayout(dataDir) # subjects subjList = layout.get_subjects() # sessions sesList = layout.get_sessions() # tasks taskList = layout.get_tasks() # runs runList = layout.get_runs() # List of all fMRI data for subject 01 fMRI_sub01 = layout.get(subject='01', suffix='bold', extension=['nii', 'nii.gz'], return_type='file') # Lets focus on test session fMRI_sub01_test = layout.get(subject='01', session='test', suffix='bold',
from bids.layout import BIDSLayout #from nipype import Workflow, Node, Function, MapNode from niworkflows.interfaces.bids import DerivativesDataSink import os # Set project variables and set up workflow project_dir = os.path.abspath('../..') data_dir = 'data/ds000171/' #wf = Workflow(name='wf', base_dir = os.getcwd()) # Leverage BIDS to get subject list layout = BIDSLayout(os.path.join(project_dir, data_dir)) subject_list = layout.get_subjects() a_path = layout.get(subject='control01', suffix='T1w')[0].path dsink = DerivativesDataSink() dsink.inputs.base_directory = os.getcwd() dsink.inputs.desc = 'preprocessed' dsink.inputs.out_path_base = "fmri-rep" dsink.inputs.source_file = a_path dsink.inputs.in_file = a_path res = dsink.run() #wf.connect(grab_anat, 'T1w', dsink, 'source_file') #wf.connect(grab_anat, 'T1w', dsink, 'in_file')
def collect_participants(bids_dir, participant_label=None, strict=False, bids_validate=True): """ List the participants under the BIDS root and checks that participants designated with the participant_label argument exist in that folder. Returns the list of participants to be finally processed. Requesting all subjects in a BIDS directory root: >>> collect_participants(str(datadir / 'ds114'), bids_validate=False) ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10'] Requesting two subjects, given their IDs: >>> collect_participants(str(datadir / 'ds114'), participant_label=['02', '04'], ... bids_validate=False) ['02', '04'] Requesting two subjects, given their IDs (works with 'sub-' prefixes): >>> collect_participants(str(datadir / 'ds114'), participant_label=['sub-02', 'sub-04'], ... bids_validate=False) ['02', '04'] Requesting two subjects, but one does not exist: >>> collect_participants(str(datadir / 'ds114'), participant_label=['02', '14'], ... bids_validate=False) ['02'] >>> collect_participants( ... str(datadir / 'ds114'), participant_label=['02', '14'], ... strict=True, bids_validate=False) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): fmriprep.utils.bids.BIDSError: ... """ if isinstance(bids_dir, BIDSLayout): layout = bids_dir else: layout = BIDSLayout(str(bids_dir), validate=bids_validate) all_participants = set(layout.get_subjects()) # Error: bids_dir does not contain subjects if not all_participants: raise BIDSError( "Could not find participants. Please make sure the BIDS data " "structure is present and correct. Datasets can be validated online " "using the BIDS Validator (http://bids-standard.github.io/bids-validator/).\n" "If you are using Docker for Mac or Docker for Windows, you " 'may need to adjust your "File sharing" preferences.', bids_dir, ) # No --participant-label was set, return all if not participant_label: return sorted(all_participants) if isinstance(participant_label, str): participant_label = [participant_label] # Drop sub- prefixes participant_label = [ sub[4:] if sub.startswith("sub-") else sub for sub in participant_label ] # Remove duplicates participant_label = sorted(set(participant_label)) # Remove labels not found found_label = sorted(set(participant_label) & all_participants) if not found_label: raise BIDSError( "Could not find participants [{}]".format( ", ".join(participant_label)), bids_dir, ) # Warn if some IDs were not found notfound_label = sorted(set(participant_label) - all_participants) if notfound_label: exc = BIDSError( "Some participants were not found: {}".format( ", ".join(notfound_label)), bids_dir, ) if strict: raise exc warnings.warn(exc.msg, BIDSWarning) return found_label
def myfunc(datadir): start = time.time() bl = BIDSLayout(datadir, exclude='sub-(?!A00010893)(.*)$') dur = time.time() - start print("{0}:{1}".format(len(bl.get_subjects()), dur))