def test_force_index(layout_ds005):
data_dir = join(get_test_data_path(), 'ds005')
target = join(data_dir, 'models', 'ds-005_type-test_model.json')
indexer = BIDSLayoutIndexer(force_index=['models'])
model_layout = BIDSLayout(data_dir, validate=True, indexer=indexer)
assert target not in layout_ds005.files
assert target in model_layout.files
assert 'all' not in model_layout.get_subjects()
for f in model_layout.files.values():
assert 'derivatives' not in f.path
def create_datasource_indiv_params(output_query,
data_dir,
indiv_params,
subjects=None,
sessions=None,
acquisitions=None,
reconstructions=None):
""" Create a datasource node that have iterables following BIDS format,
including a indiv_params file"""
bids_datasource = pe.Node(interface=BIDSDataGrabberParams(indiv_params),
name='bids_datasource')
bids_datasource.inputs.base_dir = data_dir
bids_datasource.inputs.output_query = output_query
layout = BIDSLayout(data_dir)
# Verbose
print("BIDS layout:", layout)
print("\t", layout.get_subjects())
print("\t", layout.get_sessions())
if subjects is None:
subjects = layout.get_subjects()
if sessions is None:
sessions = layout.get_sessions()
iterables = []
iterables.append(('subject', subjects))
iterables.append(('session', sessions))
if acquisitions is not None:
iterables.append(('acquisition', acquisitions))
if reconstructions is not None:
iterables.append(('reconstruction', reconstructions))
bids_datasource.iterables = iterables
return bids_datasource
def create_datasource_noT1(data_dir,
subjects=None,
sessions=None,
acquisitions=None):
""" Create a datasource node that have iterables following BIDS format """
bids_datasource = pe.Node(interface=nio.BIDSDataGrabber(),
name='bids_datasource')
bids_datasource.inputs.base_dir = data_dir
bids_datasource.inputs.output_query = {
'T1': {
"datatype": "anat",
"suffix": "T1w",
"extensions": ["nii", ".nii.gz"]
}
}
layout = BIDSLayout(data_dir)
# Verbose
print("BIDS layout:", layout)
print("\t", layout.get_subjects())
print("\t", layout.get_sessions())
if subjects is None:
subjects = layout.get_subjects()
if sessions is None:
sessions = layout.get_sessions()
iterables = []
iterables.append(('subject', subjects))
iterables.append(('session', sessions))
if acquisitions is not None:
iterables.append(('acquisition', acquisitions))
bids_datasource.iterables = iterables
return bids_datasource
def get_files():
with open('dset_config.json', 'r') as fo:
CONFIG = json.load(fo)
DATA_DIR = op.abspath('/home/data/nbc/external-datasets/ds001491/')
all_info = {}
for dset_name in list(CONFIG.keys())[:3]:
layout = BIDSLayout(op.join(DATA_DIR, dset_name))
cfg = CONFIG[dset_name]
task = cfg['task']
dset_info = {}
for sub in layout.get_subjects():
runs = layout.get_runs(subject=sub, task=task)
sub_info = {}
for run in runs:
run_info = {}
run_info['files'] = []
run_info['echo_times'] = []
for echo in sorted(
layout.get_echoes(subject=sub, task=task, run=run)):
raw_files = layout.get(subject=sub,
task=task,
run=run,
echo=echo,
extensions='.nii.gz')
preproc_files = layout.get(subject=sub,
task=task,
run=run,
root='afni-step1',
echo=echo,
extensions='.nii.gz',
desc='realign')
preproc_files = raw_files[:]
if len(preproc_files) != 1:
print(preproc_files)
raise Exception('BAD')
# Replace filename with path when using new version of bids
run_info['files'].append(preproc_files[0].filename)
metadata = layout.get_metadata(preproc_files[0].filename)
run_info['echo_times'].append(metadata['EchoTime'])
sub_info[run] = run_info
dset_info[sub] = sub_info
all_info[dset_name] = dset_info
with open('all_files.json', 'w') as fo:
json.dump(all_info, fo, indent=4, sort_keys=True)
def _run_interface(self, runtime):
from bids.layout import BIDSLayout
layout = BIDSLayout(self.inputs.bids_dir, derivatives=True)
entities = []
extensions = ['preproc_bold.nii.gz']
for subject in np.sort(layout.get_subjects()):
file = layout.get(subject=subject,
task=layout.get_tasks(),
extensions=extensions)
if file == []:
pass
else:
entity = {'subject': subject} #, 'session': session}
entities.append(entity)
self._results['entities'] = entities
return runtime
def get_files():
DSET_DIR = op.abspath('/home/data/nbc/external-datasets/ds001491/')
layout = BIDSLayout(DSET_DIR)
layout.add_derivatives(
'/home/data/nbc/external-datasets/ds001491/derivatives/afni-step1/')
task = 'images'
info = {}
for sub in sorted(layout.get_subjects()):
print(sub)
sub_info = {'files': [], 'echo_times': []}
for echo in sorted(layout.get_echoes(subject=sub, task=task)):
raw_files = layout.get(subject=sub,
task=task,
echo=echo,
extensions='.nii.gz')
preproc_files = layout.get(subject=sub,
task=task,
root='afni-step1',
extensions='.nii.gz')
# For some reason pybids doesn't index echo in derivatives
preproc_files = [
p for p in preproc_files
if 'echo-{0}'.format(echo) in p.filename
]
if len(preproc_files) != 1:
print(preproc_files)
raise Exception('BAD')
# Replace filename with path when using new version of bids
sub_info['files'].append(preproc_files[0].path)
metadata = raw_files[0].metadata
sub_info['echo_times'].append(metadata['EchoTime'])
info[sub] = sub_info
with open('preproc_files.json', 'w') as fo:
json.dump(info, fo, indent=4, sort_keys=True)
print(line)
if line == '' and process.poll() is not None:
break
if process.returncode != 0:
raise Exception("Non zero return code: {0}\n"
"{1}\n\n{2}".format(process.returncode, command,
process.stdout.read()))
if __name__ == '__main__':
dset_dir = '/home/data/nbc/Sutherland_HIVCB/dset/'
layout = BIDSLayout(dset_dir)
log_files = glob('*.log')
log_niis = [lf.replace('.log', '') for lf in log_files]
cmd = ('/home/data/nbc/tools/deface/mri_deface {0} '
'/home/data/nbc/tools/deface/talairach_mixed_with_skull.gca '
'/home/data/nbc/tools/deface/face.gca {0}')
subjects = layout.get_subjects()
for subj in subjects:
# Anatomical scans
scans = layout.get(subject=subj, extensions='nii.gz', type='T1w')
for scan in scans:
f = scan.filename
if not any([ln in f for ln in log_niis]):
scan_cmd = cmd.format(f)
run(scan_cmd)
def runpipeline(parser):
# Parse inputs
args = parser.parse_args()
print(args)
# 1) parse required inputs
bids_dir = args.bids_dir
participant = args.participant_label
# 2) parse optional inputs
nthreads = int(args.nthreads)
bet_thr = float(args.bet_thr)
small_fov = bool(args.small_fov)
read_task_SNR = bool(args.taskSNR)
# 2a) Need BIDS directory if no subjects chosen, use BIDSDataGrabber for this
layout = BIDSLayout(bids_dir)
if args.subjects:
subject_id = [s for s in args.subjects.split(",")]
print(subject_id)
else:
subject_id = layout.get_subjects()
# 2b) set directories for results
deriv_dir = op.join(op.realpath(bids_dir), "derivatives")
# Set work & crash directories
if args.work_dir:
work_dir = op.realpath(args.work_dir)
crash_dir = op.join(op.realpath(args.work_dir), "crash")
else:
work_dir = op.join(bids_dir, "derivatives/work")
crash_dir = op.join(op.join(op.realpath(bids_dir), "derivatives/work"),
"crash")
if len(subject_id) == 1:
work_dir = op.join(work_dir, subject_id[0])
crash_dir = op.join(work_dir, "crash")
if not op.exists(work_dir):
os.makedirs(work_dir)
if not op.exists(crash_dir):
os.makedirs(crash_dir)
# 2c) set output directories
if args.out_dir:
out_dir = op.realpath(args.out_dir)
else:
out_dir = op.join(deriv_dir, "phaseprep")
config.update_config({
"logging": {
"log_directory": work_dir,
"log_to_file": True,
},
"execution": {
"crashdump_dir": crash_dir,
"crashfile_format": "txt",
"hash_method": "content",
"remove_unnecessary_outputs": False,
},
})
logging.update_logging(config)
phaseprep = pe.Workflow(name="phaseprep")
phaseprep.base_dir = work_dir
sink_dict = {}
infosource = pe.Node(interface=ul.IdentityInterface(fields=["subject_id"]),
name="infosource")
infosource.iterables = [("subject_id", subject_id)]
filegrabber = pe.Node(
ul.Function(
function=get_magandphase,
input_names=["bids_dir", "subject_id"],
output_names=["maglist", "phaselist"],
),
name="filegrabber",
)
filegrabber.inputs.bids_dir = bids_dir
phaseprep.connect([(infosource, filegrabber, [("subject_id", "subject_id")
])])
# Step two will be magnitude preprocessing
preproc_mag_wf = create_preprocess_mag_wf()
preproc_mag_wf.inputs.inputspec.frac = bet_thr
preproc_mag_wf.inputs.extractor.robust = small_fov
sink_dict["procmag"] = pe.MapNode(
DerivativesDataSink(
desc="procmag",
out_path_base="phaseprep",
base_directory="out_dir",
compress=True,
),
name="dsink_procmag",
iterfield=["in_file", "source_file"],
)
phaseprep.connect([
(
filegrabber,
preproc_mag_wf,
[
("maglist", "inputspec.input_mag"),
(("maglist", get_tasklength), "inputspec.task"),
(("maglist", get_restlength), "inputspec.rest"),
],
),
(
preproc_mag_wf,
sink_dict["procmag"],
[("outputspec.proc_mag", "in_file")],
),
(filegrabber, sink_dict["procmag"], [("maglist", "source_file")]),
])
# Step three will be phase preprocessing
preproc_phase_wf = create_preprocess_phase_wf()
sink_dict["procphase"] = sink_dict["procmag"].clone("procphase")
sink_dict["procphase"].inputs.desc = "procphase"
phaseprep.connect([
(
filegrabber,
preproc_phase_wf,
[
("phaselist", "inputspec.input_phase"),
("maglist", "inputspec.input_mag"),
(("phaselist", get_tasklength), "inputspec.task"),
(("phaselist", get_restlength), "inputspec.rest"),
],
),
(
preproc_mag_wf,
preproc_phase_wf,
[
("outputspec.motion_par", "inputspec.motion_par"),
("outputspec.mask_file", "inputspec.mask_file"),
],
),
(
preproc_phase_wf,
sink_dict["procphase"],
[("outputspec.proc_phase", "in_file")],
),
(filegrabber, sink_dict["procphase"], [("phaselist", "source_file")]),
])
# Regress ge magnitude and phase
phaseregress = pe.MapNode(
interface=PhaseFitOdr.PhaseFitOdr(),
name="phaseregressodr",
iterfield=["phase", "mag", "TR"],
)
phaseregress.iterables = ("noise_lb", [0.1, 0.15, 0.25, 0.4])
phaseregress.inputs.n_threads = 1
sink_dict["micro"] = sink_dict["procmag"].clone("micro")
sink_dict["micro"].inputs.desc = "micro"
sink_dict["macro"] = sink_dict["procmag"].clone("macro")
sink_dict["macro"].inputs.desc = "macro"
sink_dict["r2"] = sink_dict["procmag"].clone("r2")
sink_dict["r2"].inputs.desc = "r2"
sink_dict["beta"] = sink_dict["procmag"].clone("beta")
sink_dict["beta"].inputs.desc = "beta"
phaseprep.connect([
(
preproc_mag_wf,
phaseregress,
[
("outputspec.proc_mag", "mag"),
(("outputspec.proc_mag", get_TR), "TR"),
],
),
(preproc_phase_wf, phaseregress, [("outputspec.proc_phase", "phase")]),
(phaseregress, sink_dict["macro"], [("filt", "in_file")]),
(filegrabber, sink_dict["macro"], [("maglist", "source_file")]),
(phaseregress, sink_dict["micro"], [("sim", "in_file")]),
(filegrabber, sink_dict["micro"], [("maglist", "source_file")]),
(phaseregress, sink_dict["r2"], [("corr", "in_file")]),
(filegrabber, sink_dict["r2"], [("maglist", "source_file")]),
(phaseregress, sink_dict["beta"], [("beta", "in_file")]),
(filegrabber, sink_dict["beta"], [("maglist", "source_file")]),
])
# if tcompcor is true run with tcompcor applied as well
if args.tcompcor:
tcompcor = pe.MapNode(interface=TCompCor(),
name="tcompcor",
iterfield=["realigned_file"])
stripheader_tcompcor = pe.MapNode(
interface=ul.Function(
function=stripheader,
input_names=["filename"],
output_names=["new_filename"],
),
name="stripheader_tcompcor",
iterfield=["filename"],
)
phaseregress_multi = pe.MapNode(
interface=PhaseFitOdr.PhaseFitOdr(),
name="phaseregress_multi",
iterfield=["phase", "mag", "global_regressors", "TR"],
)
phaseregress_multi.inputs.n_threads = 1
phaseregress_multi.iterables = ("noise_lb", [0.1, 0.15, 0.25, 0.4])
sink_dict["micro_tcomp"] = sink_dict["procmag"].clone("micro_tcomp")
sink_dict["micro_tcomp"].inputs.desc = "micro_tcomp"
sink_dict["macro_tcomp"] = sink_dict["procmag"].clone("macro_tcomp")
sink_dict["macro_tcomp"].inputs.desc = "macro_tcomp"
sink_dict["r2_tcomp"] = sink_dict["procmag"].clone("r2_tcomp")
sink_dict["r2_tcomp"].inputs.desc = "r2_tcomp"
sink_dict["beta_tcomp"] = sink_dict["procmag"].clone("beta_tcomp")
sink_dict["beta_tcomp"].inputs.desc = "beta_tcomp"
phaseprep.connect([
(
preproc_mag_wf,
tcompcor,
[
("outputspec.proc_mag", "realigned_file"),
("outputspec.mask_file", "mask_files"),
],
),
(tcompcor, stripheader_tcompcor, [("components_file", "filename")
]),
(
stripheader_tcompcor,
phaseregress_multi,
[("new_filename", "global_regressors")],
),
(
preproc_mag_wf,
phaseregress_multi,
[
("outputspec.proc_mag", "mag"),
(("outputspec.proc_mag", get_TR), "TR"),
],
),
(
preproc_phase_wf,
phaseregress_multi,
[("outputspec.proc_phase", "phase")],
),
(phaseregress_multi, sink_dict["macro_tcomp"], [("filt", "in_file")
]),
(filegrabber, sink_dict["macro_tcomp"], [("maglist", "source_file")
]),
(phaseregress_multi, sink_dict["micro_tcomp"], [("sim", "in_file")
]),
(filegrabber, sink_dict["micro_tcomp"], [("maglist", "source_file")
]),
(phaseregress_multi, sink_dict["r2_tcomp"], [("corr", "in_file")]),
(filegrabber, sink_dict["r2_tcomp"], [("maglist", "source_file")]),
(phaseregress, sink_dict["beta_tcomp"], [("beta", "in_file")]),
(filegrabber, sink_dict["beta_tcomp"], [("maglist", "source_file")
]),
])
# Step five will be ongoing during the previous steps ensuring correct sinking
# Step six will be running this into a report
print("setup pipline succesfully")
if not args.test:
print("running pipeline")
starttime = time.time()
phaseprep.write_graph(format="png")
phaseprep.run(plugin="MultiProc", plugin_args={"n_procs": nthreads})
print("completed pipeline in ", time.time() - starttime, " seconds.")
def eval_multiclass_metrics_dataset(dataset_name,
man_analysis_name="manual_segmentation",
suffix="space-orig_desc-brain_dseg",
output_dir="evaluation_results"):
data_dir = os.path.join(data_path, dataset_name)
layout = BIDSLayout(data_dir)
# Verbose
print("BIDS layout:", layout)
subjects = layout.get_subjects()
sessions = layout.get_sessions()
print(subjects)
print(sessions)
res_eval_path = os.path.join(data_dir, "derivatives", output_dir)
try:
os.makedirs(res_eval_path)
except OSError:
print("res_eval_path {} already exists".format(res_eval_path))
results = []
for sub in subjects:
if sub in ["032139", "032141", "032142", "032143"]:
continue
for ses in sessions:
print("**** Running multiclass sub {} ses {} ****".format(
sub, ses))
man_mask_file = os.path.join(
data_dir, "derivatives", man_analysis_name,
"sub-{}".format(sub), "ses-{}".format(ses), "anat",
"sub-{}_ses-{}_{}.nii.gz".format(sub, ses, suffix))
assert os.path.exists(
man_mask_file), "Error, could not find file {}".format(
man_mask_file)
for auto_analysis_name in auto_analysis_names:
auto_mask_file = os.path.join(
data_dir, "derivatives", auto_analysis_name,
"sub-{}".format(sub), "ses-{}".format(ses), "anat",
"sub-{}_ses-{}_{}.nii.gz".format(sub, ses, suffix))
assert os.path.exists(
auto_mask_file), "Error, could not find file {}".format(
auto_mask_file)
eval_name = "manual-{}".format(auto_analysis_name)
print("Comparing multiclass {} and {}".format(
man_analysis_name, auto_analysis_name))
list_res = compute_all_multiclass_metrics(
man_mask_file, auto_mask_file)
list_res.insert(0, eval_name)
list_res.insert(0, ses)
list_res.insert(0, sub)
results.append(list_res)
#sub, ses, eval_name, VP, FP, VN, FN, kappa, dice, JC
df = pd.DataFrame(results,
columns=[
"Subject", "Session", "Evaluation", "ICC", "VP",
"FP", "VN", "FN", "Kappa", "Dice", "Jaccard"
])
csv_name = "multiclass_" + dataset_name + "_eval_res.csv"
df.to_csv(os.path.join(res_eval_path, csv_name))
return df
ValueError: BIDS root does not exist: /tmp/tmp9g7ddldw/bids-grabber/examples/examples/BIDS
#problem with base_dir
bg.inputs.base_dir = "/mnt/Filbey/Evan/MJXProcessing/examples/examples/BIDS"
bg.inputs.subject = 'M7500516'
#bg.inputs.output_query = {'T1w': dict(type='anat')}
res = bg.run()
res.outputs
print("done")
"""
def printMe(paths):
print("\n\nanalyzing " + str(paths) + "\n\n")
analyzeANAT = Node(Function(function=printMe, input_names=["paths"],
output_names=[]), name="analyzeANAT")
bg_all = Node(BIDSDataGrabber(), name='bids-grabber')
bg_all.inputs.base_dir = '/mnt/Filbey/Evan/MJXProcessing/examples/examples/BIDS'
bg_all.inputs.output_query = {'ses': dict(type='session')}
bg_all.iterables = ('subject', layout.get_subjects()[0])
wf = Workflow(name="bids_demo")
wf.connect(bg_all, "session", analyzeANAT, "paths")
wf.run()
class events_class(object):
def __init__(self, base_dir, task_id, sub_ids=None, verbose=True):
assert type(base_dir) == str, "base_dir should be type(str)"
assert type(task_id) == str, "task_id should be type(str)"
self.base_dir = base_dir
self.layout = BIDSLayout(base_dir)
self.task_id = task_id
if sub_ids == None:
self.sub_ids = self.layout.get_subjects(task=task_id)
else:
assert type(sub_ids) == list, "sub_ids should be type(list)"
for i in sub_ids:
assert type(
i) == str, "elements in sub_ids should be type(str)"
assert any(i == j
for j in self.layout.get_subjects(task=task_id)
), f"cannot find sub_ids with {task} task data"
if verbose:
print(f'{len(self.sub_ids)} subjects in {self.task_id} task')
def get_timing(self,
trial_type_cols=[],
trimTRby=10,
software='AFNI',
write=True):
'''
Throws timing information for each subject into dictionary
'''
assert (software == 'AFNI'
or software == 'SPM'), "software should be AFNI or SPM"
self.events_dict = {}
self.events_dict[self.task_id] = {}
for sub_identifier in self.sub_ids:
self.events_dict[self.task_id][sub_identifier] = {}
run_ids = self.layout.get_runs(subject=sub_identifier,
task=self.task_id)
for run_identifier in run_ids:
__eventFile__ = self.layout.get(suffix='events',
task=self.task_id,
run=run_identifier,
extension='tsv',
return_type='file')
# get df of events information
__trialInfo__ = pd.read_table(__eventFile__[0])
if run_identifier == 1:
onsets = {}
durations = {}
if trial_type_cols == []: #if no columns for trial_types specified
for column in __trialInfo__.columns[2:]:
__conditions__ = sorted(
list(set(__trialInfo__[column])))
# extracting onset and duration information for task, subject, run, trial_type
if run_identifier == 1:
onsets[column] = {}
durations[column] = {}
for itrial in __conditions__:
if run_identifier == 1:
onsets[column][itrial] = {}
durations[column][itrial] = {}
try:
onsets[column][itrial][run_identifier] = (
list(__trialInfo__[__trialInfo__[column] ==
itrial].onset -
trimTRby)
) # subtracting trimTRby due to remove dummy scans
durations[column][itrial][run_identifier] = (
list(__trialInfo__[__trialInfo__[column] ==
itrial].duration))
except KeyError:
onsets[column][itrial] = {}
durations[column][itrial] = {}
onsets[column][itrial][run_identifier] = (
list(__trialInfo__[__trialInfo__[column] ==
itrial].onset -
trimTRby)
) # subtracting trimTRby due to remove dummy scans
durations[column][itrial][run_identifier] = (
list(__trialInfo__[__trialInfo__[column] ==
itrial].duration))
else:
for t1 in trial_type_cols:
assert any(t1 == t2 for t2 in __trialInfo__.columns
), f"{t1} info is not in *events.tsv"
for column in trial_type_cols:
__conditions__ = sorted(
list(set(__trialInfo__[column])))
# extracting onset and duration information for task, subject, run, trial_type
if run_identifier == 1:
onsets[column] = {}
durations[column] = {}
for itrial in __conditions__:
if run_identifier == 1:
onsets[column][itrial] = {}
durations[column][itrial] = {}
try:
onsets[column][itrial][run_identifier] = (
list(__trialInfo__[__trialInfo__[column] ==
itrial].onset -
trimTRby)
) # subtracting trimTRby due to remove dummy scans
durations[column][itrial][run_identifier] = (
list(__trialInfo__[__trialInfo__[column] ==
itrial].duration))
except KeyError:
onsets[column][itrial] = {}
durations[column][itrial] = {}
onsets[column][itrial][run_identifier] = (
list(__trialInfo__[__trialInfo__[column] ==
itrial].onset -
trimTRby)
) # subtracting trimTRby due to remove dummy scans
durations[column][itrial][run_identifier] = (
list(__trialInfo__[__trialInfo__[column] ==
itrial].duration))
self.events_dict[self.task_id][sub_identifier]['onsets'] = onsets
self.events_dict[
self.task_id][sub_identifier]['durations'] = durations
if write == True:
if trial_type_cols == []: #if no columns for trial_types specified
for column in __trialInfo__.columns[2:]:
if column == 'response_time': #cannot handle response_times yet
continue
writeToPath = os.path.join(self.base_dir,
'derivatives', 'timing',
f'sub-{sub_identifier}',
column)
if not os.path.exists(writeToPath):
os.makedirs(writeToPath)
for (onset_key, onset_val), (dur_key, dur_val) in zip(
onsets[column].items(),
durations[column].items()):
assert (
onset_key == dur_key
), "onset conditions and duration conditions do not match!"
with open(
f'{writeToPath}/sub-{sub_identifier}_task-{self.task_id}_timing-{onset_key}.txt',
'w') as filehandle:
for (run_onsets,
run_durs) in zip(onset_val.values(),
dur_val.values()):
filehandle.writelines(
"%f:%f\t" % (onset_time, dur_time)
for (onset_time, dur_time
) in zip(run_onsets, run_durs))
filehandle.write("\n")
else:
for column in trial_type_cols:
if column == 'response_time': #cannot handle response_times yet
continue
writeToPath = os.path.join(self.base_dir,
'derivatives', 'timing',
f'sub-{sub_identifier}',
column)
if not os.path.exists(writeToPath):
os.makedirs(writeToPath)
for (onset_key, onset_val), (dur_key, dur_val) in zip(
onsets[column].items(),
durations[column].items()):
assert (
onset_key == dur_key
), "onset conditions and duration conditions do not match!"
with open(
f'{writeToPath}/sub-{sub_identifier}_task-{self.task_id}_timing-{onset_key}.txt',
'w') as filehandle:
for (run_onsets,
run_durs) in zip(onset_val.values(),
dur_val.values()):
filehandle.writelines(
"%f:%f\t" % (onset_time, dur_time)
for (onset_time, dur_time
) in zip(run_onsets, run_durs))
filehandle.write("\n")
def collect_participants(bids_dir, participant_label=None, strict=False):
"""
List the participants under the BIDS root and checks that participants
designated with the participant_label argument exist in that folder.
Returns the list of participants to be finally processed.
Requesting all subjects in a BIDS directory root:
>>> collect_participants('ds114')
['01', '02', '03', '04', '05', '06', '07', '08', '09', '10']
Requesting two subjects, given their IDs:
>>> collect_participants('ds114', participant_label=['02', '04'])
['02', '04']
Requesting two subjects, given their IDs (works with 'sub-' prefixes):
>>> collect_participants('ds114', participant_label=['sub-02', 'sub-04'])
['02', '04']
Requesting two subjects, but one does not exist:
>>> collect_participants('ds114', participant_label=['02', '14'])
['02']
>>> collect_participants('ds114', participant_label=['02', '14'],
... strict=True) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
fmriprep.utils.bids.BIDSError:
...
"""
layout = BIDSLayout(bids_dir)
all_participants = layout.get_subjects()
# Error: bids_dir does not contain subjects
if not all_participants:
raise BIDSError(
'Could not find participants. Please make sure the BIDS data '
'structure is present and correct. Datasets can be validated online '
'using the BIDS Validator (http://incf.github.io/bids-validator/).\n'
'If you are using Docker for Mac or Docker for Windows, you '
'may need to adjust your "File sharing" preferences.', bids_dir)
# No --participant-label was set, return all
if not participant_label:
return all_participants
# Drop sub- prefixes
participant_label = [
sub[4:] if sub.startswith('sub-') else sub for sub in participant_label
]
found_label = layout.get_subjects(subject=participant_label)
if not found_label:
raise BIDSError(
'Could not find participants [{}]'.format(
', '.join(participant_label)), bids_dir)
# Warn if some IDs were not found
notfound_label = sorted(set(participant_label) - set(found_label))
if notfound_label:
exc = BIDSError(
'Some participants were not found: {}'.format(
', '.join(notfound_label)), bids_dir)
if strict:
raise exc
warnings.warn(exc.msg, BIDSWarning)
return found_label
def eval_monoclass_metrics_dataset(dataset_name,
man_analysis_name="manual_segmentation",
suffix="space-orig_desc-brain_mask",
output_dir="evaluation_results"):
bids_dir = os.path.join(bids_path)
# was very long
layout = BIDSLayout(bids_dir)
### from https://github.com/bids-standard/pybids/pull/523
layout = BIDSLayout(
bids_dir,
database_file="bidsdb.sql",
)
# Verbose
print("BIDS layout:", layout)
all_subjects = layout.get_subjects()
all_sessions = layout.get_sessions()
print(all_subjects)
print(all_sessions)
data_dir = os.path.join(data_path, dataset_name)
res_eval_path = os.path.join(data_dir, "derivatives", output_dir)
try:
os.makedirs(res_eval_path)
except OSError:
print("res_eval_path {} already exists".format(res_eval_path))
results = []
for sub in subjects:
assert sub in all_subjects, "Error, subject {} was not found in bids dir {}".format(
sub, all_subjects)
for ses in sessions:
assert ses in all_sessions, "Error, session {} was not found in bids dir {}".format(
ses, all_subjects)
print("**** Running monoclass sub {} ses {} ****".format(sub, ses))
man_mask_file = os.path.join(
data_dir, "derivatives", man_analysis_name,
"sub-{}".format(sub), "ses-{}".format(ses), "anat",
"sub-{}_ses-{}_{}.nii.gz".format(sub, ses, suffix))
assert os.path.exists(
man_mask_file), "Error, could not find file {}".format(
man_mask_file)
for auto_analysis_name in auto_analysis_names:
auto_mask_file = os.path.join(
data_dir, "derivatives", auto_analysis_name,
"sub-{}".format(sub), "ses-{}".format(ses), "anat",
"sub-{}_ses-{}_{}.nii.gz".format(sub, ses, suffix))
assert os.path.exists(
auto_mask_file), "Error, could not find file {}".format(
auto_mask_file)
eval_name = "manual-{}".format(auto_analysis_name)
print("Comparing monoclass {} and {}".format(
man_analysis_name, auto_analysis_name))
list_res = compute_all_monoclass_metrics(
man_mask_file,
auto_mask_file,
pref=os.path.join(res_eval_path,
sub + "_" + ses + "_" + eval_name + "_"))
list_res.insert(0, eval_name)
list_res.insert(0, ses)
list_res.insert(0, sub)
results.append(list_res)
#sub, ses, eval_name, VP, FP, VN, FN, kappa, dice, JC, LCE, GCE
df = pd.DataFrame(results,
columns=[
"Subject", "Session", "Evaluation", "VP", "FP", "VN",
"FN", "Kappa", "Dice", "Jaccard", "LCE", "GCE"
])
csv_name = "monoclass_" + dataset_name + "_eval_res.csv"
df.to_csv(os.path.join(res_eval_path, csv_name))
return df
def get_input_subjects(bids_input_folder):
"""
"""
layout = BIDSLayout(bids_input_folder)
return layout.get_subjects()
# set paths for spm:
spm.SPMCommand.set_mlab_paths(paths=path_spm, matlab_cmd=path_matlab)
MatlabCommand.set_default_paths(path_spm)
MatlabCommand.set_default_matlab_cmd(path_matlab)
sub_list = ['sub-01']
elif 'linux' in sys.platform:
# path_matlab = '/home/mpib/wittkuhn/spm12.simg eval \$SPMMCRCMD'
# path_matlab = opj('/home', 'beegfs', 'wittkuhn', 'tools', 'spm', 'spm12.simg eval \$SPMMCRCMD')
singularity_cmd = 'singularity run -B /home/mpib/wittkuhn -B /mnt/beegfs/home/wittkuhn /home/mpib/wittkuhn/highspeed/highspeed-glm/tools/spm/spm12.simg'
singularity_spm = 'eval \$SPMMCRCMD'
path_matlab = ' '.join([singularity_cmd, singularity_spm])
spm.SPMCommand.set_mlab_paths(matlab_cmd=path_matlab, use_mcr=True)
# grab the list of subjects from the bids data set:
layout = BIDSLayout(opj(path_root, 'bids'))
# get all subject ids:
sub_list = sorted(layout.get_subjects())
# create a template to add the "sub-" prefix to the ids
sub_template = ['sub-'] * len(sub_list)
# add the prefix to all ids:
sub_list = ["%s%s" % t for t in zip(sub_template, sub_list)]
# if user defined to run specific subject
sub_list = sub_list[int(sys.argv[1]):int(sys.argv[2])]
# print the SPM version:
print('using SPM version %s' % spm.SPMCommand().version)
# ======================================================================
# DEFINE PBS CLUSTER JOB TEMPLATE (NEEDED WHEN RUNNING ON THE CLUSTER):
# ======================================================================
job_template = """
#PBS -l walltime=10:00:00
#PBS -j oe
#PBS -o /home/mpib/wittkuhn/highspeed/highspeed-glm/logs/glm
def main():
parser = makeParser()
results = parser.parse_args()
if results.boutiques:
createDescriptor(parser, results)
return 0
verb = results.verbose
fsldir = results.fsldir
mni152 = op.join(fsldir, "data", "standard", "MNI152_T1_2mm_brain.nii.gz")
mni152bn = op.basename(mni152).split(".")[0]
outdir = results.output_dir
partis = results.participant_label
labels = results.parcellation
if verb:
print("BIDS Dir: {0}".format(results.bids_dir), flush=True)
print("Output Dir: {0}".format(results.output_dir), flush=True)
print("Analysis level: {0}".format(results.analysis_level), flush=True)
# This preprocessing workflow is modified from the FSL recommendations here:
# https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/FDT/UserGuide
# Step 0, 1: Begin interrogation of BIDS dataset
# Due to current super-linear slowdown in BIDS Layout, exclude all but
# participant of interest. Explored in the following Github issue:
# https://github.com/bids-standard/pybids/issues/285
if partis is not None:
pattrn = 'sub-(?!{0})(.*)$'.format("|".join(partis))
else:
pattrn = ''
dset = BIDSLayout(results.bids_dir, exclude=pattrn)
subjects = dset.get_subjects()
if results.participant_label is not None:
subjects = [pl for pl in results.participant_label if pl in subjects]
assert (len(subjects) > 0)
if verb:
print("Participants: {0}".format(", ".join(s for s in subjects)),
flush=True)
sessions = dset.get_sessions()
if results.session_label is not None:
sessions = [sl for sl in results.session_label if sl in sessions]
assert (len(sessions) > 0)
if verb:
print("Sessions: {0}".format(", ".join(s for s in sessions)),
flush=True)
# Step 0, 2: Prune dataset to subjects/sessions that have necessary files
ftypes = [".nii", ".bval", ".bvec"]
collections = []
for subj in subjects:
for sess in sessions:
tf_dwi = dset.get(subject=subj,
session=sess,
datatype="dwi",
suffix="dwi",
return_type="file")
tf_anat = dset.get(subject=subj,
session=sess,
datatype="anat",
suffix="T1w",
return_type="file")
if (all(any(ftype in fl for fl in tf_dwi) for ftype in ftypes)
and any(ftypes[0] in fl for fl in tf_anat)):
collections += [{
"subject":
subj,
"session":
sess,
"anat": [t for t in tf_anat if ftypes[0] in t][0],
"bval": [t for t in tf_dwi if ftypes[1] in t][0],
"bvec": [t for t in tf_dwi if ftypes[2] in t][0],
"dwi": [t for t in tf_dwi if ftypes[0] in t][0]
}]
else:
if verb:
print("Skipping sub-{0}".format(subj) +
" / ses-{0} due to missing data.".format(sess),
flush=True)
complete_collection = []
for col in collections:
dwibn = op.basename(col["dwi"]).split('.')[0]
anatbn = op.basename(col["anat"]).split('.')[0]
subses = op.join('sub-{0}'.format(col['subject']),
'ses-{0}'.format(col['session']))
derivdir_d = op.join(outdir, subses, "dwi")
derivdir_a = op.join(outdir, subses, "anat")
execute("mkdir -p {0}".format(derivdir_d),
verbose=verb,
skipif=op.isdir(derivdir_d))
execute("mkdir -p {0}".format(derivdir_a),
verbose=verb,
skipif=op.isdir(derivdir_a))
# Step 1: Extract B0 volumes
# Make even number of spatial voxels? (req'd for eddy for some reason)
# TODO : above, if actually needed - docs inconsistent
# Get B0 locations
with open(col["bval"]) as fhandle:
bvals = fhandle.read().split(" ")
bvals = [int(b) for b in bvals if b != '' and b != '\n']
b0_loc = [i for i, b in enumerate(bvals) if b == np.min(bvals)]
# Get B0 volumes
col["b0_scans"] = []
for idx, b0 in enumerate(b0_loc):
b0ind = "b0_{0}".format(idx)
col["b0_scans"] += [
op.join(derivdir_d, dwibn + "_" + b0ind + ".nii.gz")
]
execute(fsl.fslroi(col["dwi"], col["b0_scans"][-1], *[b0, 1]),
verbose=verb,
skipif=op.isfile(col["b0_scans"][-1]))
# Merge B0 volumes
col["b0s"] = op.join(derivdir_d, dwibn + "_b0s.nii.gz")
execute(fsl.fslmerge(col["b0s"], *col["b0_scans"]),
verbose=verb,
skipif=op.isfile(col["b0s"]))
# Create acquisition parameters file
col["acqparams"] = op.join(derivdir_d, dwibn + "_acq.txt")
acqs = {
"i": "1 0 0",
"i-": "-1 0 0",
"j": "0 1 0",
"j-": "0 -1 0",
"k": "0 0 1",
"k-": "0 0 -1"
}
with open(col["acqparams"], 'w') as fhandle:
meta = dset.get_metadata(path=col["dwi"])
pedir = meta["PhaseEncodingDirection"]
trout = meta["TotalReadoutTime"]
line = "{0} {1}".format(acqs[pedir], trout)
fhandle.write("\n".join([line] * len(b0_loc)))
# Step 1.5: Run Top-up on Diffusion data
# TODO: remove; topup only applies with multiple PEs (rare in open data)
# col["topup"] = op.join(derivdir_d, dwibn + "_topup")
# col["hifi_b0"] = op.join(derivdir_d, dwibn + "_hifi_b0")
# execute(fsl.topup(col["b0s"], col["acqparams"],
# col["topup"], col["hifi_b0"]),
# verbose=verb)
# execute(fsl.fslmaths(col["hifi_b0"], "-Tmean", col["hifi_b0"]),
# verbose=verb)
# Step 2: Brain extraction
# ... Diffusion:
col["dwi_brain"] = op.join(derivdir_d, dwibn + "_brain.nii.gz")
col["dwi_mask"] = op.join(derivdir_d, dwibn + "_brain_mask.nii.gz")
execute(fsl.bet(col["dwi"], col["dwi_brain"], "-F", "-m"),
verbose=verb,
skipif=op.isfile(col["dwi_brain"]))
# ... Structural:
col["anat_brain"] = op.join(derivdir_a, anatbn + "_brain.nii.gz")
col["anat_mask"] = op.join(derivdir_a, anatbn + "_brain.nii.gz")
execute(fsl.bet(col["anat"], col["anat_brain"], "-m"),
verbose=verb,
skipif=op.isfile(col["anat_brain"]))
# Step 3: Produce prelimary DTIfit QC figures
col["dwi_qc_pre"] = op.join(derivdir_d, dwibn + "_dtifit_pre")
execute(fsl.dtifit(col["dwi_brain"], col["dwi_qc_pre"],
col["dwi_mask"], col["bvec"], col["bval"]),
verbose=verb,
skipif=op.isfile(col["dwi_qc_pre"] + "_FA.nii.gz"))
# Step 4: Perform eddy correction
# ... Create index
col["index"] = op.join(derivdir_d, dwibn + "_eddy_index.txt")
with open(col["index"], 'w') as fhandle:
fhandle.write(" ".join(["1"] * len(bvals)))
# ... Run eddy
col["eddy_dwi"] = op.join(derivdir_d, dwibn + "_eddy")
if results.gpu:
eddy_exe = "eddy_cuda8.0"
else:
eddy_exe = "eddy_openmp"
execute(fsl.eddy(col["dwi_brain"],
col["dwi_mask"],
col["acqparams"],
col["index"],
col["bvec"],
col["bval"],
col["eddy_dwi"],
exe=eddy_exe),
verbose=verb,
skipif=op.isfile(col["eddy_dwi"] + ".nii.gz"))
# Step 5: Registration to template
# ... Compute transforms
col["t1w2mni"] = op.join(derivdir_a, anatbn + "_to_mni_xfm.mat")
execute(fsl.flirt(col["anat_brain"], omat=col["t1w2mni"], ref=mni152),
verbose=verb,
skipif=op.isfile(col["t1w2mni"]))
col["dwi2t1w"] = op.join(derivdir_d, dwibn + "_to_t1w_xfm.mat")
execute(fsl.flirt(col["eddy_dwi"],
ref=col["anat_brain"],
omat=col["dwi2t1w"]),
verbose=verb,
skipif=op.isfile(col["dwi2t1w"]))
col["dwi2mni"] = op.join(derivdir_d, dwibn + "_to_mni_xfm.mat")
execute(fsl.convert_xfm(concat=col["t1w2mni"],
inp=col["dwi2t1w"],
omat=col["dwi2mni"]),
verbose=verb,
skipif=op.isfile(col["dwi2mni"]))
# ... Invert transforms towards diffusion space
col["mni2dwi"] = op.join(derivdir_d, dwibn + "_from_mni_xfm.mat")
execute(fsl.convert_xfm(inverse=col["dwi2mni"], omat=col["mni2dwi"]),
verbose=verb,
skipif=op.isfile(col["mni2dwi"]))
col["t1w2dwi"] = op.join(derivdir_a, anatbn + "_dwi_xfm.mat")
execute(fsl.convert_xfm(inverse=col["dwi2t1w"], omat=col["t1w2dwi"]),
verbose=verb,
skipif=op.isfile(col["t1w2dwi"]))
# Step 6: Apply registrations to anatomical and template images
col["anat_in_dwi"] = op.join(derivdir_a, anatbn + "_brain_dwi.nii.gz")
execute(fsl.flirt(col["anat_brain"],
applyxfm=True,
out=col["anat_in_dwi"],
init=col["t1w2dwi"],
ref=col["eddy_dwi"]),
verbose=verb,
skipif=op.isfile(col["anat_in_dwi"]))
col["mni_in_dwi"] = op.join(
derivdir_d, ("atlas_" + dwibn + "_" + mni152bn + "_dwi.nii.gz"))
execute(fsl.flirt(mni152,
applyxfm=True,
out=col["mni_in_dwi"],
init=col["mni2dwi"],
ref=col["eddy_dwi"]),
verbose=verb,
skipif=op.isfile(col["mni_in_dwi"]))
# Step 7: Perform tissue segmentation on anatomical images in DWI space
col["tissue_masks"] = op.join(derivdir_d, anatbn + "_fast")
execute(fsl.fast(col["anat_in_dwi"],
col["tissue_masks"],
classes=3,
imtype=1),
verbose=verb,
skipif=op.isfile(col["tissue_masks"] + "_seg_2.nii.gz"))
# Step 8: Transform parcellations into DWI space
col["labels_in_dwi"] = []
for label in labels:
lbn = op.basename(label).split('.')[0]
col["labels_in_dwi"] += [
op.join(derivdir_d,
("labels_" + dwibn + "_" + lbn + ".nii.gz"))
]
execute(fsl.flirt(label,
applyxfm=True,
out=col["labels_in_dwi"][-1],
init=col["mni2dwi"],
ref=col["eddy_dwi"],
interp="nearestneighbour"),
verbose=verb,
skipif=op.isfile(col["labels_in_dwi"][-1]))
if verb:
print("Finished processing sub-{0}".format(subj) +
" / ses-{0} !".format(sess),
flush=True)
complete_collection += [col]
def makeInput(self, inp):
"""
Obtains the input (file-paths and input-mode) from the main window
and updates the subjects/time-series objects appropriately
"""
# obtains the Raw BOLD data and preprocessed data, and updates the data-store
# each input comprises of an input_file, mask_file and a file_type, and the mode
input_file, mask_file, file_type, mode = inp
# if the mode is file (that is input is a stand-alone NIfTI or GIfTI file)
if mode == "file":
return self.makeInputFile(input_file, mask_file, file_type, 'file')
# if the input is present in a directory organized in BIDS format
elif "bids" in mode:
# obtaining all the input files
layout = BIDSLayout(input_file,
validate=False,
config=['bids', 'derivatives'])
subjects_to_analyze = layout.get_subjects()
# if no subjects were found
if not subjects_to_analyze:
return Status(
False,
error=
'Could not find participants. Please make sure the BIDS data '
'structure is present and correct. Datasets can be validated online '
'using the BIDS Validator (http://incf.github.io/bids-validator/).'
)
all_inputs = layout.get(return_type='filename',
datatype='func',
subject=subjects_to_analyze,
task='rest',
desc='preproc',
suffix='bold',
extension=['nii', 'nii.gz'])
# if no *bold.nii or *bold.nii.gz files were found
if not all_inputs != []:
return Status(
False,
error='There are no files of type *bold.nii / *bold.nii.gz '
'Please make sure to have at least one file of the above type '
'in the BIDS specification')
# if the atlas file is provided
if mode == "bids w/ atlas":
for file_count in range(len(all_inputs)):
# making input for every combination of input-file and atlas
temp = (self.makeInputFile(all_inputs[file_count],
mask_file, ".nii",
'bids w/ atlas'))
if not temp.get_state():
return temp
# returns the status
return Status(
True,
info="Preprocessed all input",
dic={"Number of Input Subjects: ": len(all_inputs)})
# if the atlas file is not provided
elif mode == "bids":
# obtaining the masks
all_masks = layout.get(return_type='filename',
datatype='func',
subject=subjects_to_analyze,
task='rest',
desc='brain',
suffix='mask',
extension=['nii', 'nii.gz'])
# if no mask files were found
if not all_masks != []:
return Status(
False,
error=
'There are no files of type *mask.nii / *mask.nii.gz '
'Please make sure to have at least one file of the above type '
'in the BIDS specification')
# if the number of mask files != number of input files
if len(all_inputs) != len(all_masks):
return Status(
False,
error=
'The number of *bold.nii / .nii.gz and the number of '
'*mask.nii / .nii.gz are different. Please make sure that '
'there is one mask for each input_file present')
all_inputs.sort()
all_masks.sort()
# matching the prefix to align the inputs and the masks together
all_prefix_match = False
prefix_match_count = 0
for i in range(len(all_inputs)):
input_prefix = all_inputs[i].split('/')[-1].split(
'_desc')[0]
mask_prefix = all_masks[i].split('/')[-1].split('_desc')[0]
if input_prefix == mask_prefix:
prefix_match_count += 1
else:
all_prefix_match = False
break
if prefix_match_count == len(all_inputs):
all_prefix_match = True
# if the mask and input files have different prefix
if not all_prefix_match:
return Status(
False,
error=
'The mask and input files should have the same prefix for correspondence. '
'Please consider renaming your files')
else:
# making input for every combination of input-file and mask-file
for file_count in range(len(all_inputs)):
temp = (self.makeInputFile(all_inputs[file_count],
all_masks[file_count],
".nii", 'bids'))
if not temp.get_state():
return temp
# returns the status
return Status(
True,
info="Preprocessed all input",
dic={"Number of Input Subjects: ": len(all_inputs)})
fig.tight_layout()
fig.savefig(op.join(out_dir, 'scrubbing_analysis.png'), dpi=400)
del mean_delta_r, scrub_smc, perm_scrub_smc
if __name__ == '__main__':
"""
Run a test using 40 subjects from the ADHD dataset.
"""
dset_dir = '/home/data/nbc/Sutherland_ACE/dset/'
deriv_dir = '/home/data/nbc/Sutherland_ACE/derivatives/fmriprep-1.2.1/'
fd_thresh = 0.2
n_iters = 10000
layout = BIDSLayout(dset_dir)
subjects = layout.get_subjects(task='rest')
motpars = []
files = []
for sub in subjects:
preproc_file = op.join(
deriv_dir,
'sub-{0}/ses-S1/func/sub-{0}_ses-S1_task-rest_run-01_space-'
'MNI152NLin2009cAsym_desc-preproc_bold.nii.gz'.format(sub))
files.append(preproc_file)
conf_file = op.join(
deriv_dir,
'sub-{0}/ses-S1/func/sub-{0}_ses-S1_task-rest_run-01_desc-'
'confounds_regressors.tsv'.format(sub))
conf = pd.read_csv(conf_file, sep='\t')
motpars_ = conf[[
'trans_x', 'trans_y', 'trans_z', 'rot_x', 'rot_y', 'rot_z'
def run_rsHRF():
parser = get_parser()
args = parser.parse_args()
arg_groups = {}
for group in parser._action_groups:
group_dict = {
a.dest: getattr(args, a.dest, None)
for a in group._group_actions
}
arg_groups[group.title] = group_dict
para = arg_groups['Parameters']
nargs = len(sys.argv)
temporal_mask = []
if (not args.GUI) and (args.output_dir is None):
parser.error(
'--output_dir is required when executing in command-line interface'
)
if (not args.GUI) and (args.estimation is None):
parser.error(
'--estimation rule is required when executing in command-line interface'
)
if (args.GUI):
if (nargs == 2):
try:
from .rsHRF_GUI import run
run.run()
except ModuleNotFoundError:
parser.error(
'--GUI should not be used inside a Docker container')
else:
parser.error('--no other arguments should be supplied with --GUI')
if (args.input_file is not None
or args.ts is not None) and args.analysis_level:
parser.error(
'analysis_level cannot be used with --input_file or --ts, do not supply it'
)
if (args.input_file is not None
or args.ts is not None) and args.participant_label:
parser.error(
'participant_labels are not to be used with --input_file or --ts, do not supply it'
)
if args.input_file is not None and args.brainmask:
parser.error(
'--brainmask cannot be used with --input_file, use --atlas instead'
)
if args.ts is not None and (args.brainmask or args.atlas):
parser.error(
'--atlas or --brainmask cannot be used with --ts, do not supply it'
)
if args.bids_dir is not None and not (args.brainmask or args.atlas):
parser.error(
'--atlas or --brainmask needs to be supplied with --bids_dir')
if args.bids_dir is not None and not args.analysis_level:
parser.error(
'analysis_level needs to be supplied with bids_dir, choices=[participant]'
)
if args.input_file is not None and (not args.input_file.endswith(
('.nii', '.nii.gz', '.gii', '.gii.gz'))):
parser.error(
'--input_file should end with .gii, .gii.gz, .nii or .nii.gz')
if args.atlas is not None and (not args.atlas.endswith(
('.nii', '.nii.gz', '.gii', '.gii.gz'))):
parser.error('--atlas should end with .gii, .gii.gz, .nii or .nii.gz')
if args.ts is not None and (not args.ts.endswith(('.txt'))):
parser.error('--ts file should end with .txt')
if args.temporal_mask is not None and (not args.temporal_mask.endswith(
('.dat'))):
parser.error('--temporal_mask ile should end with ".dat"')
if args.temporal_mask is not None:
f = open(args.temporal_mask, 'r')
for line in f:
for each in line:
if each in ['0', '1']:
temporal_mask.append(int(each))
if args.estimation == 'sFIR' or args.estimation == 'FIR':
para['T'] = 1
if args.ts is not None:
file_type = op.splitext(args.ts)
if para['TR'] <= 0:
parser.error('Please supply a valid TR using -TR argument')
else:
TR = para['TR']
para['dt'] = para['TR'] / para['T']
para['lag'] = np.arange(np.fix(para['min_onset_search'] / para['dt']),
np.fix(para['max_onset_search'] / para['dt']) +
1,
dtype='int')
fourD_rsHRF.demo_rsHRF(args.ts,
None,
args.output_dir,
para,
args.n_jobs,
file_type,
mode='time-series',
temporal_mask=temporal_mask,
wiener=args.wiener)
if args.input_file is not None:
if args.atlas is not None:
if (args.input_file.endswith(
('.nii', '.nii.gz')) and args.atlas.endswith(
('.gii', '.gii.gz'))) or (args.input_file.endswith(
('.gii', '.gii.gz')) and args.atlas.endswith(
('.nii', '.nii.gz'))):
parser.error(
'--atlas and input_file should be of the same type [NIfTI or GIfTI]'
)
# carry analysis with input_file and atlas
file_type = op.splitext(args.input_file)
if file_type[-1] == ".gz":
file_type = op.splitext(file_type[-2])[-1] + file_type[-1]
else:
file_type = file_type[-1]
if ".nii" in file_type:
TR = (spm_dep.spm.spm_vol(args.input_file).header.get_zooms())[-1]
else:
if para['TR'] == -1:
parser.error('Please supply a valid TR using -TR argument')
else:
TR = para['TR']
if TR <= 0:
if para['TR'] <= 0:
parser.error('Please supply a valid TR using -TR argument')
else:
if para['TR'] == -1:
para['TR'] = TR
elif para['TR'] <= 0:
print('Invalid TR supplied, using implicit TR: {0}'.format(TR))
para['TR'] = TR
para['dt'] = para['TR'] / para['T']
para['lag'] = np.arange(np.fix(para['min_onset_search'] / para['dt']),
np.fix(para['max_onset_search'] / para['dt']) +
1,
dtype='int')
fourD_rsHRF.demo_rsHRF(args.input_file,
args.atlas,
args.output_dir,
para,
args.n_jobs,
file_type,
mode='input',
temporal_mask=temporal_mask,
wiener=args.wiener)
if args.bids_dir is not None:
utils.bids.write_derivative_description(args.bids_dir, args.output_dir)
bids_dir = Path(args.bids_dir)
fname = bids_dir / 'dataset_description.json'
if fname.exists():
desc = json.loads(Path(fname).read_text())
if 'DataType' in desc:
if desc['DataType'] != 'derivative':
parser.error(
'Input data is not a derivative dataset'
' (DataType in dataset_description.json is not equal to "derivative")'
)
else:
parser.error(
'DataType is not defined in the dataset_description.json file. Please make sure DataType is defined. '
'Information on the dataset_description.json file can be found online '
'(https://bids-specification.readthedocs.io/en/stable/03-modality-agnostic-files.html'
'#derived-dataset-and-pipeline-description)')
else:
parser.error(
'Could not find dataset_description.json file. Please make sure the BIDS data '
'structure is present and correct. Datasets can be validated online '
'using the BIDS Validator (http://incf.github.io/bids-validator/).'
)
if args.bids_dir is not None and args.atlas is not None:
# carry analysis with bids_dir and 1 atlas
layout = BIDSLayout(args.bids_dir,
validate=False,
config=['bids', 'derivatives'])
if args.participant_label:
input_subjects = args.participant_label
subjects_to_analyze = layout.get_subjects(subject=input_subjects)
else:
subjects_to_analyze = layout.get_subjects()
if not subjects_to_analyze:
parser.error(
'Could not find participants. Please make sure the BIDS data '
'structure is present and correct. Datasets can be validated online '
'using the BIDS Validator (http://incf.github.io/bids-validator/).'
)
if not args.atlas.endswith(('.nii', '.nii.gz')):
parser.error('--atlas should end with .nii or .nii.gz')
if args.bids_filter_file is not None:
filter_list = json.loads(Path(args.bids_filter_file).read_text())
default_input = {
'extension': 'nii.gz',
'datatype': 'func',
'desc': 'preproc',
'task': 'rest',
'suffix': 'bold'
}
default_input['subject'] = subjects_to_analyze
default_input.update(filter_list['bold'])
all_inputs = layout.get(return_type='filename', **default_input)
else:
all_inputs = layout.get(return_type='filename',
datatype='func',
subject=subjects_to_analyze,
task='rest',
desc='preproc',
suffix='bold',
extension=['nii', 'nii.gz'])
if not all_inputs != []:
parser.error(
'There are no files of type *bold.nii / *bold.nii.gz '
'Please make sure to have at least one file of the above type '
'in the BIDS specification')
else:
num_errors = 0
for file_count in range(len(all_inputs)):
try:
TR = layout.get_metadata(
all_inputs[file_count])['RepetitionTime']
except KeyError as e:
TR = spm_dep.spm.spm_vol(
all_inputs[file_count]).header.get_zooms()[-1]
para['TR'] = TR
para['dt'] = para['TR'] / para['T']
para['lag'] = np.arange(
np.fix(para['min_onset_search'] / para['dt']),
np.fix(para['max_onset_search'] / para['dt']) + 1,
dtype='int')
num_errors += 1
try:
fourD_rsHRF.demo_rsHRF(all_inputs[file_count],
args.atlas,
args.output_dir,
para,
args.n_jobs,
file_type,
mode='bids w/ atlas',
temporal_mask=temporal_mask,
wiener=args.wiener)
num_errors -= 1
except ValueError as err:
print(err.args[0])
except:
print("Unexpected error:", sys.exc_info()[0])
success = len(all_inputs) - num_errors
if success == 0:
raise RuntimeError(
'Dimensions were inconsistent for all input-mask pairs; \n'
'No inputs were processed!')
if args.bids_dir is not None and args.brainmask:
# carry analysis with bids_dir and brainmask
layout = BIDSLayout(args.bids_dir,
validate=False,
config=['bids', 'derivatives'])
if args.participant_label:
input_subjects = args.participant_label
subjects_to_analyze = layout.get_subjects(subject=input_subjects)
else:
subjects_to_analyze = layout.get_subjects()
if not subjects_to_analyze:
parser.error(
'Could not find participants. Please make sure the BIDS data '
'structure is present and correct. Datasets can be validated online '
'using the BIDS Validator (http://incf.github.io/bids-validator/).'
)
if args.bids_filter_file is not None:
filter_list = json.loads(Path(args.bids_filter_file).read_text())
default_input = {
'extension': 'nii.gz',
'datatype': 'func',
'desc': 'preproc',
'task': 'rest',
'suffix': 'bold'
}
default_input['subject'] = subjects_to_analyze
default_input.update(filter_list['bold'])
all_inputs = layout.get(return_type='filename', **default_input)
default_mask = {
'extension': 'nii.gz',
'datatype': 'func',
'desc': 'brain',
'task': 'rest',
'suffix': 'mask'
}
default_mask['subject'] = subjects_to_analyze
default_mask.update(filter_list['mask'])
all_masks = layout.get(return_type='filename', **default_mask)
else:
all_inputs = layout.get(return_type='filename',
datatype='func',
subject=subjects_to_analyze,
task='rest',
desc='preproc',
suffix='bold',
extension=['nii', 'nii.gz'])
all_masks = layout.get(return_type='filename',
datatype='func',
subject=subjects_to_analyze,
task='rest',
desc='brain',
suffix='mask',
extension=['nii', 'nii.gz'])
if not all_inputs != []:
parser.error(
'There are no files of type *bold.nii / *bold.nii.gz '
'Please make sure to have at least one file of the above type '
'in the BIDS specification')
if not all_masks != []:
parser.error(
'There are no files of type *mask.nii / *mask.nii.gz '
'Please make sure to have at least one file of the above type '
'in the BIDS specification')
if len(all_inputs) != len(all_masks):
parser.error(
'The number of *bold.nii / .nii.gz and the number of '
'*mask.nii / .nii.gz are different. Please make sure that '
'there is one mask for each input_file present')
all_inputs.sort()
all_masks.sort()
all_prefix_match = False
prefix_match_count = 0
for i in range(len(all_inputs)):
input_prefix = all_inputs[i].split('/')[-1].split('_desc')[0]
mask_prefix = all_masks[i].split('/')[-1].split('_desc')[0]
if input_prefix == mask_prefix:
prefix_match_count += 1
else:
all_prefix_match = False
break
if prefix_match_count == len(all_inputs):
all_prefix_match = True
if not all_prefix_match:
parser.error(
'The mask and input files should have the same prefix for correspondence. '
'Please consider renaming your files')
else:
num_errors = 0
for file_count in range(len(all_inputs)):
file_type = all_inputs[file_count].split('bold')[1]
if file_type == ".nii" or file_type == ".nii.gz":
try:
TR = layout.get_metadata(
all_inputs[file_count])['RepetitionTime']
except KeyError as e:
TR = spm_dep.spm.spm_vol(
all_inputs[file_count]).header.get_zooms()[-1]
para['TR'] = TR
else:
spm_dep.spm.spm_vol(all_inputs[file_count])
TR = spm_dep.spm.spm_vol(
all_inputs[file_count]).get_arrays_from_intent(
"NIFTI_INTENT_TIME_SERIES")[0].meta.get_metadata(
)["TimeStep"]
para['TR'] = float(TR) * 0.001
para['dt'] = para['TR'] / para['T']
para['lag'] = np.arange(
np.fix(para['min_onset_search'] / para['dt']),
np.fix(para['max_onset_search'] / para['dt']) + 1,
dtype='int')
num_errors += 1
try:
fourD_rsHRF.demo_rsHRF(all_inputs[file_count],
all_masks[file_count],
args.output_dir,
para,
args.n_jobs,
mode='bids',
temporal_mask=temporal_mask,
wiener=args.wiener)
num_errors -= 1
except ValueError as err:
print(err.args[0])
except:
print("Unexpected error:", sys.exc_info()[0])
success = len(all_inputs) - num_errors
if success == 0:
raise RuntimeError(
'Dimensions were inconsistent for all input-mask pairs; \n'
'No inputs were processed!')
def read_bids_dataset(bids_input,
subject_list=None,
session_list=None,
collect_on_subject=False):
"""
extracts and organizes relevant metadata from a bids dataset necessary
for the dcan-modified hcp fmri processing pipeline.
:param bids_input: path to input bids folder
:param subject_list: EITHER, a list of subject ids to filter on,
OR a dictionary of subject id: list of sessions to filter on.
:param session_list: a list of session ids to filter on.
:param collect_on_subject: collapses all sessions, for cases with
non-longitudinal data spread across scan sessions.
:return: bids data struct (nested dict)
spec:
{
t1w: t1w filename list,
t2w: t2w filename list,
t1w_metadata: bids meta data (first t1),
t2w_metadata: bids meta data (first t2),
func: fmri filename list,
func_metadata: bids meta data list,
fmap: {
positive: spin echo filename list (if applicable)
negative: spin echo filename list (if applicable)
},
fmap_metadata: {
positive: bids meta data list (if applicable)
negative: bids meta data list (if applicable)
},
}
"""
layout = BIDSLayout(bids_input, index_metadata=True)
subjects = layout.get_subjects()
# filter subject list
if isinstance(subject_list, list):
subjects = [s for s in subjects if s in subject_list]
elif isinstance(subject_list, dict):
subjects = [s for s in subjects if s in subject_list.keys()]
subsess = []
# filter session list
for s in subjects:
sessions = layout.get_sessions(subject=s)
# filter sessions_list
if isinstance(session_list, list):
sessions = [t for t in sessions if t in session_list]
if not sessions:
subsess += [(s, None)]
elif collect_on_subject:
subsess += [(s, sessions)]
else:
subsess += list(product([s], sessions))
assert len(subsess), 'bids data not found for participants. If labels ' \
'were provided, check the participant labels for errors. ' \
'Otherwise check that the bids folder provided is correct.'
for subject, sessions in subsess:
# get relevant image datatypes
anat, anat_types = set_anatomicals(layout, subject, sessions)
func, func_types = set_functionals(layout, subject, sessions)
fmap, fmap_types = set_fieldmaps(layout, subject, sessions)
bids_data = {
'subject': subject,
'session': sessions if not collect_on_subject else None,
'types': anat_types.union(func_types, fmap_types)
}
bids_data.update(anat)
bids_data.update(func)
bids_data.update(fmap)
yield bids_data
def find_data(self, subject_ids=None, visit_ids=None):
"""
Return subject and session information for a project in the local
repository
Parameters
----------
subject_ids : list(str)
List of subject IDs with which to filter the tree with. If None all
are returned
visit_ids : list(str)
List of visit IDs with which to filter the tree with. If None all
are returned
Returns
-------
project : arcana.repository.Tree
A hierarchical tree of subject, session and fileset information for
the repository
"""
filesets = []
layout = BIDSLayout(self.root_dir)
all_subjects = layout.get_subjects()
all_visits = layout.get_sessions()
if not all_visits:
all_visits = [self.DEFAULT_VISIT_ID]
self._depth = 1
else:
self._depth = 2
for item in layout.get(return_type='object'):
if item.path.startswith(self.derivatives_dir):
# We handle derivatives using the BasicRepo base
# class methods
continue
if not hasattr(item, 'entities') or not item.entities.get('suffix',
False):
logger.warning("Skipping unrecognised file '{}' in BIDS tree"
.format(op.join(item.dirname, item.filename)))
continue # Ignore hidden file
try:
subject_ids = [item.entities['subject']]
except KeyError:
# If item exists in top-levels of in the directory structure
# it is inferred to exist for all subjects in the tree
subject_ids = all_subjects
try:
visit_ids = [item.entities['session']]
except KeyError:
# If item exists in top-levels of in the directory structure
# it is inferred to exist for all visits in the tree
visit_ids = all_visits
for subject_id in subject_ids:
for visit_id in visit_ids:
aux_files = {}
metadata = layout.get_metadata(item.path)
if metadata and not item.path.endswith('.json'):
# Write out the combined JSON side cars to a temporary
# file to include in extended NIfTI filesets
metadata_path = op.join(
self.metadata_dir,
'sub-{}'.format(subject_id),
'ses-{}'.format(visit_id),
item.filename + '.json')
os.makedirs(op.dirname(metadata_path), exist_ok=True)
if not op.exists(metadata_path):
with open(metadata_path, 'w') as f:
json.dump(metadata, f)
aux_files['json'] = metadata_path
fileset = BidsFileset(
path=op.join(item.dirname, item.filename),
type=item.entities['suffix'],
subject_id=subject_id, visit_id=visit_id,
repository=self,
modality=item.entities.get('modality', None),
task=item.entities.get('task', None),
aux_files=aux_files)
filesets.append(fileset)
# Get derived filesets, fields and records using the same method using
# the method in the BasicRepo base class
derived_filesets, fields, records = super().find_data(
subject_ids=subject_ids, visit_ids=visit_ids)
filesets.extend(derived_filesets)
return filesets, fields, records
import os
from bids.layout import BIDSLayout
# Directory where your data set resides.
dataDir = '/tmp/Data/ds114'
# Creating the layout object for this BIDS data set
layout = BIDSLayout(dataDir)
# subjects
subjList = layout.get_subjects()
# sessions
sesList = layout.get_sessions()
# tasks
taskList = layout.get_tasks()
# runs
runList = layout.get_runs()
# List of all fMRI data for subject 01
fMRI_sub01 = layout.get(subject='01',
suffix='bold',
extension=['nii', 'nii.gz'],
return_type='file')
# Lets focus on test session
fMRI_sub01_test = layout.get(subject='01',
session='test',
suffix='bold',
from bids.layout import BIDSLayout
#from nipype import Workflow, Node, Function, MapNode
from niworkflows.interfaces.bids import DerivativesDataSink
import os
# Set project variables and set up workflow
project_dir = os.path.abspath('../..')
data_dir = 'data/ds000171/'
#wf = Workflow(name='wf', base_dir = os.getcwd())
# Leverage BIDS to get subject list
layout = BIDSLayout(os.path.join(project_dir, data_dir))
subject_list = layout.get_subjects()
a_path = layout.get(subject='control01', suffix='T1w')[0].path
dsink = DerivativesDataSink()
dsink.inputs.base_directory = os.getcwd()
dsink.inputs.desc = 'preprocessed'
dsink.inputs.out_path_base = "fmri-rep"
dsink.inputs.source_file = a_path
dsink.inputs.in_file = a_path
res = dsink.run()
#wf.connect(grab_anat, 'T1w', dsink, 'source_file')
#wf.connect(grab_anat, 'T1w', dsink, 'in_file')
def collect_participants(bids_dir,
participant_label=None,
strict=False,
bids_validate=True):
"""
List the participants under the BIDS root and checks that participants
designated with the participant_label argument exist in that folder.
Returns the list of participants to be finally processed.
Requesting all subjects in a BIDS directory root:
>>> collect_participants(str(datadir / 'ds114'), bids_validate=False)
['01', '02', '03', '04', '05', '06', '07', '08', '09', '10']
Requesting two subjects, given their IDs:
>>> collect_participants(str(datadir / 'ds114'), participant_label=['02', '04'],
... bids_validate=False)
['02', '04']
Requesting two subjects, given their IDs (works with 'sub-' prefixes):
>>> collect_participants(str(datadir / 'ds114'), participant_label=['sub-02', 'sub-04'],
... bids_validate=False)
['02', '04']
Requesting two subjects, but one does not exist:
>>> collect_participants(str(datadir / 'ds114'), participant_label=['02', '14'],
... bids_validate=False)
['02']
>>> collect_participants(
... str(datadir / 'ds114'), participant_label=['02', '14'],
... strict=True, bids_validate=False) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
fmriprep.utils.bids.BIDSError:
...
"""
if isinstance(bids_dir, BIDSLayout):
layout = bids_dir
else:
layout = BIDSLayout(str(bids_dir), validate=bids_validate)
all_participants = set(layout.get_subjects())
# Error: bids_dir does not contain subjects
if not all_participants:
raise BIDSError(
"Could not find participants. Please make sure the BIDS data "
"structure is present and correct. Datasets can be validated online "
"using the BIDS Validator (http://bids-standard.github.io/bids-validator/).\n"
"If you are using Docker for Mac or Docker for Windows, you "
'may need to adjust your "File sharing" preferences.',
bids_dir,
)
# No --participant-label was set, return all
if not participant_label:
return sorted(all_participants)
if isinstance(participant_label, str):
participant_label = [participant_label]
# Drop sub- prefixes
participant_label = [
sub[4:] if sub.startswith("sub-") else sub for sub in participant_label
]
# Remove duplicates
participant_label = sorted(set(participant_label))
# Remove labels not found
found_label = sorted(set(participant_label) & all_participants)
if not found_label:
raise BIDSError(
"Could not find participants [{}]".format(
", ".join(participant_label)),
bids_dir,
)
# Warn if some IDs were not found
notfound_label = sorted(set(participant_label) - all_participants)
if notfound_label:
exc = BIDSError(
"Some participants were not found: {}".format(
", ".join(notfound_label)),
bids_dir,
)
if strict:
raise exc
warnings.warn(exc.msg, BIDSWarning)
return found_label
def myfunc(datadir):
start = time.time()
bl = BIDSLayout(datadir, exclude='sub-(?!A00010893)(.*)$')
dur = time.time() - start
print("{0}:{1}".format(len(bl.get_subjects()), dur))
