def get_bids_files(input_bids_dir,
space="MNI152NLin2009cAsym",
subject=bids.layout.Query(2),
validate=False,
save_cache=False,
load_cache=False):
'''Extract the mask, and imaging entities from the fmriprep output'''
database_path = os.path.join(input_bids_dir, ".pybids_cache")
if load_cache:
layout = bids.BIDSLayout(input_bids_dir,
validate=validate,
database_path=database_path)
else:
layout = bids.BIDSLayout(input_bids_dir, validate=validate)
layout.add_derivatives(input_bids_dir)
if save_cache:
if os.path.exists(database_path):
shutil.rmtree(database_path)
layout.save(database_path)
bids_images = layout.get(scope="derivatives",
space=space,
subject=subject,
desc="preproc",
suffix=["T1w", "bold"],
extension="nii.gz")
bids_masks = layout.get(scope="derivatives",
space=space,
subject=subject,
desc="brain",
suffix="mask",
extension="nii.gz")
return bids_images, bids_masks
def __init__(self,
bids_dir,
selected_pipeline,
bids_filter=None,
bidsvalidator=False,
update_pipeline=True,
history=None,
exist_ok=False,
layout=None,
nettsv='nn-t'):
if layout is None:
self.BIDSLayout = bids.BIDSLayout(bids_dir,
derivatives=True,
validate=bidsvalidator)
else:
self.BIDSLayout = layout
self.bids_dir = bids_dir
self.selected_pipeline = selected_pipeline
self.nettsv = nettsv
self.bids_filter = {} if bids_filter is None else bids_filter
if history is not None:
self.history = {}
self.exist_ok = exist_ok
self.update_pipeline = update_pipeline
with open(tenetopath[0] +
'/config/tenetobids/tenetobids_description.json') as f:
self.tenetobids_description = json.load(f)
self.tenetobids_description['PipelineDescription'][
'Version'] = tenetoversion
with open(tenetopath[0] +
'/config/tenetobids/tenetobids_structure.json') as f:
self.tenetobids_structure = json.load(f)
def main():
args = parse_args()
pybids_cache_path = os.path.join(args.bids_path, PYBIDS_CACHE_PATH)
layout = bids.BIDSLayout(
args.bids_path,
database_path=pybids_cache_path,
reset_database=args.force_reindex,
ignore=(
"code",
"stimuli",
"sourcedata",
"models",
re.compile(r"^\."),
) + load_bidsignore(args.bids_path),
)
job_path = os.path.join(layout.root, SLURM_JOB_DIR)
if not os.path.exists(job_path):
os.mkdir(job_path)
# add .slurm to .gitignore
with open(os.path.join(layout.root, ".gitignore"), "a+") as f:
f.seek(0)
if not any([SLURM_JOB_DIR in l for l in f.readlines()]):
f.write(f"{SLURM_JOB_DIR}\n")
for job_file in run_mriqc(layout, args, args.preproc):
if not args.no_submit:
submit_slurm_job(job_file)
def _list_outputs(self):
import bids
base_dir = self.inputs.base_directory
os.makedirs(base_dir, exist_ok=True)
layout = bids.BIDSLayout(base_dir)
path_patterns = self.inputs.path_patterns
if not isdefined(path_patterns):
path_patterns = None
out_files = []
for entities, in_file in zip(self.inputs.entities,
self.inputs.in_file):
ents = {**self.inputs.fixed_entities}
ents.update(entities)
ents = {k: snake_to_camel(str(v)) for k, v in ents.items()}
out_fname = os.path.join(base_dir,
layout.build_path(ents, path_patterns))
makedirs(os.path.dirname(out_fname), exist_ok=True)
_copy_or_convert(in_file, out_fname)
out_files.append(out_fname)
return {'out_file': out_files}
def _run_interface(self, runtime):
import bids
from bids.analysis import auto_model
models = self.inputs.model
if not isinstance(models, list):
# model is not yet standardized, so validate=False
layout = bids.BIDSLayout(self.inputs.bids_dir, validate=False)
if not isdefined(models):
models = layout.get(suffix='smdl', return_type='file')
if not models:
raise ValueError("No models found")
elif models == 'default':
models = auto_model(layout)
models = [_ensure_model(m) for m in models]
if self.inputs.selectors:
# This is almost certainly incorrect
models = [
model for model in models
if all(val in model['input'].get(key, [val])
for key, val in self.inputs.selectors.items())
]
self._results['model_spec'] = models
return runtime
def _run_interface(self, runtime):
import bids
bids.config.set_options(loop_preproc=True)
include = self.inputs.include_pattern
exclude = self.inputs.exclude_pattern
if not isdefined(include):
include = None
if not isdefined(exclude):
exclude = None
paths = [(self.inputs.bids_dir, 'bids')]
if isdefined(self.inputs.preproc_dir):
paths.append((self.inputs.preproc_dir, ['bids', 'derivatives']))
layout = bids.BIDSLayout(paths, include=include, exclude=exclude)
selectors = self.inputs.selectors
analysis = bids.Analysis(model=self.inputs.model, layout=layout)
analysis.setup(drop_na=False, **selectors)
self._load_level1(runtime, analysis)
self._load_higher_level(runtime, analysis)
# Debug - remove, eventually
runtime.analysis = analysis
return runtime
def _run_interface(self, runtime):
import bids
from bids.analysis import auto_model
models = self.inputs.model
if not isinstance(models, list):
database_path = self.inputs.database_path
layout = bids.BIDSLayout.load(database_path=database_path)
if not isdefined(models):
# model is not yet standardized, so validate=False
# Ignore all subject directories and .git/ and .datalad/ directories
small_layout = bids.BIDSLayout(
layout.root, derivatives=[d.root for d in layout.derivatives.values()],
validate=False,
ignore=[re.compile(r'sub-'),
re.compile(r'\.(git|datalad)')])
# PyBIDS can double up, so find unique models
models = list(set(small_layout.get(suffix='smdl', return_type='file')))
if not models:
raise ValueError("No models found")
elif models == 'default':
models = auto_model(layout)
models = [_ensure_model(m) for m in models]
if self.inputs.selectors:
# This is almost certainly incorrect
models = [model for model in models
if all(val in model['input'].get(key, [val])
for key, val in self.inputs.selectors.items())]
self._results['model_spec'] = models
return runtime
def main():
args = parse_args()
print("\n### Running fmriprep-slurm\n")
print(vars(args))
print("\n# Loading pyBIDS database (it might take few hours for a big dataset)...\n")
sing_bids_path = os.path.join(
SINGULARITY_DATA_PATH, os.path.basename(args.bids_path))
layout = bids.BIDSLayout(
sing_bids_path,
reset_database=args.force_reindex,
ignore=(
"code",
"stimuli",
"sourcedata",
"models",
re.compile(r"^\."),
)
+ load_bidsignore(sing_bids_path),
)
job_path = os.path.join(SINGULARITY_OUTPUT_PATH, SLURM_JOB_DIR)
if not os.path.exists(job_path):
os.mkdir(job_path)
print("\n# Prefectch templateflow templates ...\n")
# prefectch templateflow templates
os.environ["TEMPLATEFLOW_HOME"] = TEMPLATEFLOW_HOME
tf_api.get(args.output_spaces + ["OASIS30ANTs", "fsLR", "fsaverage"])
print("\n# Processing slurm files into {}\n".format(
os.path.join(args.output_path, SLURM_JOB_DIR)))
for job_file in run_fmriprep(layout, args):
if args.submit:
submit_slurm_job(job_file)
def parse_directory(deriv_dir, work_dir, analysis):
fl_layout = bids.BIDSLayout((deriv_dir, [
'bids', 'derivatives',
pkgr.resource_filename('fitlins', 'data/fitlins.json')
]))
wd_layout = bids.BIDSLayout(str(Path(work_dir) / 'reportlets' / 'fitlins'))
contrast_svgs = fl_layout.get(extensions='.svg', type='contrasts')
analyses = []
for contrast_svg in contrast_svgs:
ents = fl_layout.parse_file_entities(contrast_svg.filename)
ents.pop('type')
ents.setdefault('subject', None)
correlation_matrix = fl_layout.get(extensions='.svg',
type='corr',
**ents)
design_matrix = fl_layout.get(extensions='.svg', type='design', **ents)
job_desc = {
'ents': {k: v
for k, v in ents.items() if v is not None},
'dataset': analysis.layout.root,
'model_name': analysis.model['name'],
'contrasts_svg': contrast_svg.filename,
}
if ents.get('subject'):
job_desc['subject_id'] = ents.get('subject')
if correlation_matrix:
job_desc['correlation_matrix_svg'] = correlation_matrix[0].filename
if design_matrix:
job_desc['design_matrix_svg'] = design_matrix[0].filename
snippet = wd_layout.get(extensions='.html', type='snippet', **ents)
if snippet:
with open(snippet[0].filename) as fobj:
job_desc['warning'] = fobj.read()
contrasts = fl_layout.get(extensions='.png', type='ortho', **ents)
# TODO: Split contrasts from estimates
job_desc['contrasts'] = [{
'image_file':
c.filename,
'name':
fl_layout.parse_file_entities(c.filename)['contrast']
} for c in contrasts]
analyses.append(job_desc)
return analyses
def __init__(self, bdir, subjects=None, sessions=None, pipeline='dwi'):
self.bdir = bdir
self.layout = bids.BIDSLayout(bdir)
if subjects is None:
subjects = self.layout.get_subjects()
if sessions is None:
sessions = self.layout.get_sessions()
# get list of subject / session pairs
self.pairs = self.get_pairs(subjects=subjects, sessions=sessions, pipeline=pipeline)
def query(self):
"""
Get a queryable object representing the entire Bids dataset.
Returns
-------
A BidsLayout object
"""
return bids.BIDSLayout(self.bids_root)
def idconn_workflow(dset_dir, atlas, task, out_dir, space="MNI152NLin2009cAsym", conn=None, bids_db=None, confounds=None):
print('Getting started!')
if not confounds:
confounds = [
"cosine00", "cosine01", "cosine02",
"trans_x", "trans_x_derivative1", "trans_x_power2", "trans_x_derivative1_power2",
"trans_y", "trans_y_derivative1", "trans_y_derivative1_power2", "trans_y_power2",
"trans_z", "trans_z_derivative1", "trans_z_power2", "trans_z_derivative1_power2",
"rot_x", "rot_x_derivative1", "rot_x_power2", "rot_x_derivative1_power2",
"rot_y", "rot_y_derivative1", "rot_y_power2", "rot_y_derivative1_power2",
"rot_z", "rot_z_derivative1", "rot_z_derivative1_power2", "rot_z_power2",
"a_comp_cor_00", "a_comp_cor_01", "a_comp_cor_02", "a_comp_cor_03", "a_comp_cor_04", "a_comp_cor_05", "a_comp_cor_06"
]
print(f"Atlas: {atlas}\nConnectivity measure: {conn}")
assert exists(dset_dir), f"Specified dataset doesn't exist:\n{dset_dir} not found.\n\nPlease check the filepath."
layout = bids.BIDSLayout(dset_dir, derivatives=True, database_path=bids_db)
subjects = layout.get(return_type='id', target='subject', suffix='bold')
print(f"Subjects: {subjects}")
#runs = layout.get(return_type='id', target='session', suffix='bold')
preproc_subjects = layout.get(return_type='id', target='subject', task=task, space=space, desc='preproc', suffix='bold')
if len(subjects) != len(preproc_subjects):
print(f'{len(subjects)} subjects found in dset, only {len(preproc_subjects)} have preprocessed BOLD data. Pipeline is contniuing anyway, please double check preprocessed data if this doesn\'t seem right.')
example_events = layout.get(return_type='filename', suffix='events', task=task, subject=preproc_subjects[0])
events_df = pd.read_csv(example_events[0], header=0, index_col=0, sep='\t')
conditions = events_df['trial_type'].unique()
print(f"Computing connectivity matrices using {atlas}")
for subject in preproc_subjects:
print(f"Subject {subject}")
sessions = layout.get(return_type='id', target='session', task=task, subject=subject, suffix='bold')
print(f"Sessions with task-{task} found for {subject}: {sessions}")
for session in sessions:
print(f"Session {session}")
print(f"here are the inputs: {layout, subject, session, task, atlas, conn, space, confounds}")
if 'rest' in task:
try:
adj_matrix = build_networks.connectivity(layout, subject, session, task, atlas, conn, space, confounds)
except Exception as e:
print(f'Error building corrmat for sub-{subject}, ses-{session}, task-{task}: {e}')
if len(conditions) < 1:
try:
adj_matrix = build_networks.connectivity(layout, subject, session, task, atlas, conn, space, confounds)
except Exception as e:
print(f'Error building corrmat for sub-{subject}, ses-{session}, task-{task}: {e}')
else:
try:
adj_matrix = build_networks.task_connectivity(layout=layout, subject=subject, session=session, task=task, atlas=atlas, confounds=confounds, connectivity_metric=conn)
except Exception as e:
print(f'Error building corrmat for sub-{subject}, ses-{session}, task-{task}: {e}')
def get_subjects(input_bids_dir,
validate=False,
save_cache=False,
load_cache=False):
'''Get the subjects IDs from the fmriprep output'''
database_path = os.path.join(input_bids_dir, ".pybids_cache")
if load_cache:
layout = bids.BIDSLayout(input_bids_dir,
validate=validate,
database_path=database_path)
else:
layout = bids.BIDSLayout(input_bids_dir, validate=validate)
layout.add_derivatives(input_bids_dir)
if save_cache:
if os.path.exists(database_path):
shutil.rmtree(database_path)
layout.save(database_path)
sub_ids = layout.get_subjects()
return sub_ids
def test_extension_initial_dot(mock_config):
ds117 = os.path.join(get_test_data_path(), 'ds000117')
# Warn if creating a layout without declaring a mode
bids.config.set_option('extension_initial_dot', None)
with pytest.warns(FutureWarning, match='To suppress this warning'):
layout = bids.BIDSLayout(ds117)
assert layout.get(extension='nii.gz')[0].entities['extension'] == 'nii.gz'
# Warn if setting the mode to False... this isn't sticking around
with pytest.warns(FutureWarning, match='will be disabled'):
bids.config.set_option('extension_initial_dot', False)
with pytest.warns(None) as record:
layout = bids.BIDSLayout(ds117)
assert len(record) == 0
assert layout.get(extension='nii.gz')[0].entities['extension'] == 'nii.gz'
# No warnings to move to dot mode
with pytest.warns(None) as record:
bids.config.set_option('extension_initial_dot', True)
layout = bids.BIDSLayout(ds117)
assert len(record) == 0
assert layout.get(extension='nii.gz')[0].entities['extension'] == '.nii.gz'
def test_extension_initial_dot(mock_config):
ds117 = os.path.join(get_test_data_path(), 'ds000117')
# Setting False is disabled
with pytest.raises(ValueError, match='Cannot set'):
bids.config.set_option('extension_initial_dot', False)
with pytest.raises(ValueError, match='Cannot set'):
bids.config.set_option('extension_initial_dot', None)
# Setting True warns
with pytest.warns(FutureWarning, match='will be removed'):
bids.config.set_option('extension_initial_dot', True)
# No warnings on layout construction
with pytest.warns(None) as record:
layout = bids.BIDSLayout(ds117)
assert len(record) == 0
assert layout.get(extension='nii.gz')[0].entities['extension'] == '.nii.gz'
def _run_interface(self, runtime):
import bids
paths = [(self.inputs.bids_dir, 'bids')]
if isdefined(self.inputs.preproc_dir):
paths.append((self.inputs.preproc_dir, ['bids', 'derivatives']))
layout = bids.BIDSLayout(paths)
bold_files = []
mask_files = []
entities = []
for ents in self.inputs.entities:
selectors = {**self.inputs.selectors, **ents}
bold_file = layout.get(extensions=['.nii', '.nii.gz'], **selectors)
if len(bold_file) == 0:
raise FileNotFoundError(
"Could not find BOLD file in {} with entities {}"
"".format(self.inputs.bids_dir, selectors))
elif len(bold_file) > 1:
raise ValueError(
"Non-unique BOLD file in {} with entities {}.\n"
"Matches:\n\t{}"
"".format(
self.inputs.bids_dir, selectors,
"\n\t".join('{} ({})'.format(
f.filename, layout.files[f.filename].entities)
for f in bold_file)))
# Select exactly matching mask file (may be over-cautious)
bold_ents = layout.parse_file_entities(bold_file[0].filename)
bold_ents['type'] = 'brainmask'
mask_file = layout.get(extensions=['.nii', '.nii.gz'], **bold_ents)
bold_ents.pop('type')
bold_files.append(bold_file[0].filename)
mask_files.append(mask_file[0].filename if mask_file else None)
entities.append(bold_ents)
self._results['bold_files'] = bold_files
self._results['mask_files'] = mask_files
self._results['entities'] = entities
return runtime
def write_report(level, report_dicts, run_context, deriv_dir):
fl_layout = bids.BIDSLayout(
(deriv_dir, ['bids', 'derivatives',
pkgr.resource_filename('fitlins', 'data/fitlins.json')]))
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(
searchpath=pkgr.resource_filename('fitlins', '/')))
tpl = env.get_template('data/report.tpl')
for context in report_dicts:
ents = context['ents'].copy()
ents['model'] = snake_to_camel(context['model_name'])
target_file = op.join(deriv_dir, fl_layout.build_path(ents, PATH_PATTERNS))
html = tpl.render(deroot({'level': level, **context, **run_context},
op.dirname(target_file)))
with open(target_file, 'w') as fobj:
fobj.write(html)
def bids_data(entry):
import os
import glob
import bids
import json
bids.config.set_option('extension_initial_dot', True)
layout = bids.BIDSLayout(entry.inputs,
derivatives=False,
absolute_paths=True)
if not os.path.exists(entry.outputs + '/FDT') or os.path.exists(
entry.outputs + '/FDT/' + 'dataset_description.json'):
os.makedirs(entry.outputs, exist_ok=True)
os.makedirs(entry.outputs + '/FDT', exist_ok=True)
# make dataset_description file...
import json
data = {
'Name':
'FSL Diffusion Toolbox Minimal Preprocessing',
"BIDSVersion":
"1.1.1",
"PipelineDescription": {
"Name": "FSL Diffusion Toolbox",
"Version": "0.0.1",
"CodeURL": "https://github.com/amyhegarty/FDT"
},
"CodeURL":
"https://github.com/amyhegarty/FDT",
"HowToAcknowledge":
"Please cite all relevant works for FSL tools: topup, eddy, dtifit and python tools: pybids ( https://doi.org/10.21105/joss.01294, https://doi.org/10.21105/joss.01294)"
}
with open(entry.outputs + '/FDT/' + 'dataset_description.json',
'w') as outfile:
json.dump(data, outfile, indent=2)
return layout
def main():
args = docopt(__doc__)
fmriprep_dir = args['<fmriprep_dir>']
output_dir = args['<output_dir>']
ignore_fields = args['--ignore']
space = args['--space'] or 'T1w'
layout = bids.BIDSLayout(fmriprep_dir, derivatives=True, validate=False)
#Generate participants.tsv template
participants_tsv(layout, output_dir, ignore_fields)
#Make anatomical QC pages
anat_qc = os.path.join(output_dir, 'anat')
makedir(anat_qc)
make_anatomical_qc(layout, anat_qc)
#Make functional QC pages
func_qc = os.path.join(output_dir, 'func')
makedir(func_qc)
make_functional_qc(layout, func_qc, space)
def main():
args = parse_args()
pybids_cache_path = os.path.join(args.bids_path, PYBIDS_CACHE_PATH)
layout = bids.BIDSLayout(
args.bids_path,
database_path=pybids_cache_path,
reset_database=args.force_reindex,
ignore=(
"code",
"stimuli",
"sourcedata",
"models",
re.compile(r"^\."),
) + load_bidsignore(args.bids_path),
)
job_path = os.path.join(layout.root, SLURM_JOB_DIR)
if not os.path.exists(job_path):
os.mkdir(job_path)
# add .slurm to .gitignore
with open(os.path.join(layout.root, ".gitignore"), "a+") as f:
f.seek(0)
if not any([SLURM_JOB_DIR in l for l in f.readlines()]):
f.write(f"{SLURM_JOB_DIR}\n")
# prefectch templateflow templates
os.environ["TEMPLATEFLOW_HOME"] = TEMPLATEFLOW_HOME
import templateflow.api as tf_api
tf_api.get(OUTPUT_TEMPLATES + ["OASIS30ANTs", "fsLR", "fsaverage"])
for job_file in run_fmriprep(layout, args, args.preproc):
if not args.no_submit:
submit_slurm_job(job_file)
def is_bids(input_dir):
"""
Make sure that the input data is BIDs-formatted.
If it's BIDs-formatted, except for a `dataset_description.json` file, return True.
Returns
-------
bool
True if the input directory is BIDs-formatted
Raises
------
ValueError
Occurs if the input directory is not formatted properly.
"""
try:
l = bids.BIDSLayout(input_dir)
return l.validate
except ValueError as e:
p = "'dataset_description.json' is missing from project root. Every valid BIDS dataset must have this file."
if str(e) != p:
raise ValueError(e)
create_datadescript(input_dir)
return is_bids(input_dir)
def main():
args = parse_args()
logging.basicConfig(level=logging.getLevelName(args.debug_level.upper()))
pybids_cache_path = os.path.join(args.bids_path, PYBIDS_CACHE_PATH)
layout = bids.BIDSLayout(
args.bids_path,
database_path=pybids_cache_path,
reset_database=args.force_reindex,
index_metadata=False,
validate=False,
)
if args.datalad:
annex_repo = AnnexRepo(args.bids_path)
subject_list = (
args.participant_label if args.participant_label else bids.layout.Query.ANY
)
session_list = args.session_label if args.session_label else bids.layout.Query.ANY
filters = dict(
subject=subject_list,
session=session_list,
**args.ref_bids_filters,
extension=['nii','nii.gz'])
deface_ref_images = layout.get(**filters)
if not len(deface_ref_images):
logging.info(f"no reference image found with condition {filters}")
return
new_files, modified_files = [], []
script_dir = os.path.dirname(__file__)
mni_path = os.path.abspath(os.path.join(script_dir, MNI_PATH))
mni_mask_path = os.path.abspath(os.path.join(script_dir, MNI_MASK_PATH))
# if the MNI template image is not available locally
if not os.path.exists(os.path.realpath(mni_path)):
datalad.api.get(mni_path, dataset=datalad.api.Dataset(script_dir + "/../../"))
tmpl_image = nb.load(mni_path)
tmpl_image_mask = nb.load(mni_mask_path)
tmpl_defacemask = generate_deface_ear_mask(tmpl_image)
brain_xtractor = Extractor()
for ref_image in deface_ref_images:
subject = ref_image.entities["subject"]
session = ref_image.entities["session"]
datalad.api.get(ref_image.path)
ref_image_nb = ref_image.get_image()
matrix_path = ref_image.path.replace(
"_%s.%s" % (ref_image.entities["suffix"], ref_image.entities["extension"]),
"_mod-%s_defacemaskreg.mat" % ref_image.entities["suffix"],
)
if os.path.exists(matrix_path):
logging.info("reusing existing registration matrix")
ref2tpl_affine = AffineMap(np.loadtxt(matrix_path))
else:
logging.info(f"running registration of reference serie: {ref_image.path}")
brain_mask = (brain_xtractor.run(ref_image_nb.get_fdata()) > 0.99).astype(
np.uint8
)
brain_mask[:] = scipy.ndimage.morphology.binary_dilation(
brain_mask, iterations=4
)
brain_mask_nb = nb.Nifti1Image(brain_mask, ref_image_nb.affine)
ref2tpl_affine = registration(
tmpl_image, ref_image_nb, tmpl_image_mask, brain_mask_nb
)
np.savetxt(matrix_path, ref2tpl_affine.affine)
new_files.append(matrix_path)
if args.debug_images:
output_debug_images(tmpl_image, ref_image, ref2tpl_affine)
series_to_deface = []
for filters in args.other_bids_filters:
series_to_deface.extend(
layout.get(
extension=["nii", "nii.gz"],
subject=subject,
session=session,
**filters,
)
)
# unlock before making any change to avoid unwanted save
if args.datalad:
annex_repo.unlock([serie.path for serie in series_to_deface])
for serie in series_to_deface:
if args.datalad:
if (
next(annex_repo.get_metadata(serie.path))[1].get(
"distribution-restrictions"
)
is None
):
logging.info(
f"skip {serie.path} as there are no distribution restrictions metadata set."
)
continue
logging.info(f"defacing {serie.path}")
datalad.api.get(serie.path)
serie_nb = serie.get_image()
warped_mask = warp_mask(tmpl_defacemask, serie_nb, ref2tpl_affine)
if args.save_all_masks or serie == ref_image:
warped_mask_path = serie.path.replace(
"_%s" % serie.entities["suffix"],
"_mod-%s_defacemask" % serie.entities["suffix"],
)
if os.path.exists(warped_mask_path):
logging.warning(
f"{warped_mask_path} already exists : will not overwrite, clean before rerun"
)
else:
warped_mask.to_filename(warped_mask_path)
new_files.append(warped_mask_path)
masked_serie = nb.Nifti1Image(
np.asanyarray(serie_nb.dataobj) * np.asanyarray(warped_mask.dataobj),
serie_nb.affine,
serie_nb.header,
)
masked_serie.to_filename(serie.path)
modified_files.append(serie.path)
if args.datalad and len(modified_files):
logging.info("saving files and metadata changes in datalad")
annex_repo.set_metadata(
modified_files, remove={"distribution-restrictions": "sensitive"}
)
datalad.api.save(
modified_files + new_files,
message="deface %d series/images and update distribution-restrictions"
% len(modified_files),
)
def create_layout(self):
# create bids layout
self.layout = bids.BIDSLayout(self.base_dir,
derivatives=True,
database_path=self.database_path,
reset_database=True)
def main():
args = docopt(__doc__)
fmriprep_dir = args['<fmriprep_dir>']
output_dir = args['<output_dir>']
ignore_fields = args['--ignore']
layout = bids.BIDSLayout(fmriprep_dir,
validate=False,
index_metadata=False)
# Generate participants.tsv template
participants_tsv(layout, output_dir, ignore_fields)
#Now loop through each participant's scans and start building QC pages
html_series = []
subjects = layout.get_subjects()
no_task = []
prev_task_htmls = []
for ind, s in enumerate(subjects):
broad_name = 'sub-{}_{}.html'.format(s, ind)
#Get relevant files for subject
fig_dir = os.path.join(layout.root, 'sub-{}'.format(s), 'figures')
sub_figs = os.listdir(fig_dir)
sub_files = layout.get(subject=s, extension='nii.gz')
#Get broad QC markup
broad_html = make_broad_html(s, sub_figs, sub_files, output_dir,
fig_dir)
#Write in link to the previous task html
if ind > 0:
broad_html += [add_link(prev_task_htmls[-1][0], 'Previous Page')]
#Get functional markup (one per taskfile)
task_files = layout.get(subject=s,
extension='nii.gz',
suffix='bold',
space='T1w')
task_files = sorted(task_files, key=lambda x: get_task_ordering_key(x))
task_htmls = [
make_task_html(s, output_dir, fig_dir, sub_figs, t)
for t in task_files
]
#Step 1: Write the first task_html name into broad_html
try:
broad_html += [add_link(task_htmls[0][0], 'Next Page')]
except IndexError:
no_task.append(s)
continue
#Step 2: Link broad html to first task_html
task_htmls[0][1] += [add_link(broad_name, 'Previous Page')]
#Step 3: Link each task HTML to the previous
for i in range(1, len(task_htmls)):
task_htmls[i][1] += [
add_link(task_htmls[i - 1][0], 'Previous Page')
]
#Step 4: Link each task HTML to the next
for i in range(0, len(task_htmls) - 1):
task_htmls[i][1] += [add_link(task_htmls[i + 1][0], 'Next Page')]
#Step 5: Link broad_html to the next subject
if ind != len(subjects) - 1:
broad_html += [
add_link('sub-{}_{}.html'.format(subjects[ind + 1], ind + 1),
'Next Subject')
]
task_htmls[-1][1] += [
add_link('sub-{}_{}.html'.format(subjects[ind + 1], ind + 1),
'Next Page')
]
#Write out files
with open(os.path.join(output_dir, broad_name), 'w') as f:
f.writelines(broad_html)
#Write out task files
for t in task_htmls:
with open(os.path.join(output_dir, t[0]), 'w') as f:
f.writelines(t[1])
#Store previous task htmls
prev_task_htmls = task_htmls
print('Subjects with no task files:')
print('\n'.join(no_task))
ANTS_JOBS = 8
PIPELINE_JOBS = 48
import sys, os
import re
import glob
import numpy as np
import pandas as pd
import bids
import transforms3d
import SimpleITK as sitk
from nipype.pipeline import MapNode, Workflow
from nipype.interfaces import ants
from blends import registration
ppmi = bids.BIDSLayout(DATA_PATH, validate=False)
def remove_repeats(lsScans):
# Remove repeated scans for the same subject
lsOutScans = []
lsSubjects = []
for scan in lsScans:
if scan.entities['subject'] not in lsSubjects:
lsSubjects += [scan.entities['subject']]
lsOutScans += [scan.path]
return lsOutScans
def remove_rigidbody(strAffinePath):
# Isolate and remove rigid body components (rotation and translation) from
def update_bids_layout(self):
"""
Function that upddates to new bids l
"""
self.BIDSLayout = bids.BIDSLayout(self.bids_dir, derivatives=True)
def create_bids_dataframe(self):
"""Generate the dataframe."""
# Suppress a Future Warning from pybids about leading dot included in 'extension' from version 0.14.0
# The config_bids.json file used matches the future behavior
# TODO: when reaching version 0.14.0, remove the following line
pybids.config.set_option('extension_initial_dot', True)
for path_data in self.paths_data:
path_data = os.path.join(path_data, '')
# Initialize BIDSLayoutIndexer and BIDSLayout
# validate=True by default for both indexer and layout, BIDS-validator is not skipped
# Force index for samples tsv and json files, and for subject subfolders containing microscopy files based on extensions.
# Force index of subject subfolders containing CT-scan files under "anat" or "ct" folder based on extensions and modality suffix.
# TODO: remove force indexing of microscopy files after BEP microscopy is merged in BIDS
# TODO: remove force indexing of CT-scan files after BEP CT-scan is merged in BIDS
ext_microscopy = ('.png', '.ome.tif', '.ome.tiff', '.ome.tf2',
'.ome.tf8', '.ome.btf')
ext_ct = ('.nii.gz', '.nii')
suffix_ct = ('ct', 'CT')
force_index = []
for root, dirs, files in os.walk(path_data):
for file in files:
# Microscopy
if file == "samples.tsv" or file == "samples.json":
force_index.append(file)
if (file.endswith(ext_microscopy)
and os.path.basename(root) == "microscopy" and
(root.replace(path_data, '').startswith("sub"))):
force_index.append(
os.path.join(root.replace(path_data, '')))
# CT-scan
if (file.endswith(ext_ct)
and file.split('.')[0].endswith(suffix_ct)
and (os.path.basename(root) == "anat"
or os.path.basename(root) == "ct") and
(root.replace(path_data, '').startswith("sub"))):
force_index.append(
os.path.join(root.replace(path_data, '')))
indexer = pybids.BIDSLayoutIndexer(force_index=force_index)
if self.derivatives:
self.write_derivatives_dataset_description(path_data)
layout = pybids.BIDSLayout(path_data,
config=self.bids_config,
indexer=indexer,
derivatives=self.derivatives)
# Transform layout to dataframe with all entities and json metadata
# As per pybids, derivatives don't include parsed entities, only the "path" column
df_next = layout.to_df(metadata=True)
# Add filename column
df_next.insert(1, 'filename',
df_next['path'].apply(os.path.basename))
# Drop rows with json, tsv and LICENSE files in case no extensions are provided in config file for filtering
df_next = df_next[~df_next['filename'].str.
endswith(tuple(['.json', '.tsv', 'LICENSE']))]
# Update dataframe with subject files of chosen contrasts and extensions,
# and with derivative files of chosen target_suffix from loader parameters
df_next = df_next[(
~df_next['path'].str.contains('derivatives')
& df_next['suffix'].str.contains('|'.join(self.contrast_lst))
& df_next['extension'].str.contains('|'.join(self.extensions)))
| (df_next['path'].str.contains('derivatives')
& df_next['filename'].str.
contains('|'.join(self.target_suffix)))]
if df_next[~df_next['path'].str.contains('derivatives')].empty:
# Warning if no subject files are found in path_data
logger.warning(
"No subject files were found in '{}' dataset. Skipping dataset."
.format(path_data))
else:
# Add tsv files metadata to dataframe
df_next = self.add_tsv_metadata(df_next, path_data, layout)
# TODO: check if other files are needed for EEG and DWI
# Merge dataframes
self.df = pd.concat([self.df, df_next],
join='outer',
ignore_index=True)
if self.df.empty:
# Raise error and exit if no subject files are found in any path data
raise RuntimeError(
"No subject files found. Check selection of parameters in config.json"
" and datasets compliance with BIDS specification.")
# Drop duplicated rows based on all columns except 'path'
# Keep first occurence
columns = self.df.columns.to_list()
columns.remove('path')
self.df = self.df[~(
self.df.astype(str).duplicated(subset=columns, keep='first'))]
# If indexing of derivatives is true
if self.derivatives:
# Get list of subject files with available derivatives
has_deriv, deriv = self.get_subjects_with_derivatives()
# Filter dataframe to keep subjects files with available derivatives only
if has_deriv:
self.df = self.df[
self.df['filename'].str.contains('|'.join(has_deriv))
| self.df['filename'].str.contains('|'.join(deriv))]
else:
# Raise error and exit if no derivatives are found for any subject files
raise RuntimeError("Derivatives not found.")
# Reset index
self.df.reset_index(drop=True, inplace=True)
# Drop columns with all null values
self.df.dropna(axis=1, inplace=True, how='all')
import pandas as pd
import json
import bids
import matplotlib.pyplot as plt
import plotje
# Download data from here: Esteban, O. et al. Crowdsourced MRI quality metrics
# and expert quality annotations for training of humans and machines. Sci Data 6, 30 (2019).
# Then run make_distributions.py to summarize the data from this snapshot
summary_path = './data/summary/bold_curated'
dataset = '/home/william/datasets/es-fmri_v2/'
dfd = pd.read_csv(summary_path + qc + '_summary.csv', index_col=[0])
layout = bids.BIDSLayout(dataset)
layout.add_derivatives(dataset + '/derivatives/')
layout = layout.to_df()
keeprow = []
for i, n in layout.iterrows():
if 'mriqc_output' in n['path'] and n['path'].endswith('.json'):
keeprow.append(i)
layout = layout.loc[keeprow]
layout_bold = layout[layout['suffix'] == 'bold']
params = [('pre', 'rest', 'preop'), ('es', 'es', 'postop')]
qcmet = {}
qcdesc = {}
for p in params:
qcmet[p[0]] = {}
def run_fitlins(argv=None):
import re
from nipype import logging as nlogging
warnings.showwarning = _warn_redirect
opts = get_parser().parse_args(argv)
force_index = [
# If entry looks like `/<pattern>/`, treat `<pattern>` as a regex
re.compile(ign[1:-1]) if (ign[0], ign[-1]) == ('/', '/') else ign
# Iterate over empty tuple if undefined
for ign in opts.force_index or ()
]
ignore = [
# If entry looks like `/<pattern>/`, treat `<pattern>` as a regex
re.compile(ign[1:-1]) if (ign[0], ign[-1]) == ('/', '/') else ign
# Iterate over empty tuple if undefined
for ign in opts.ignore or ()
]
log_level = 25 + 5 * (opts.quiet - opts.verbose)
logger.setLevel(log_level)
nlogging.getLogger('nipype.workflow').setLevel(log_level)
nlogging.getLogger('nipype.interface').setLevel(log_level)
nlogging.getLogger('nipype.utils').setLevel(log_level)
if not opts.space:
# make it an explicit None
opts.space = None
if not opts.desc_label:
# make it an explicit None
opts.desc_label = None
ncpus = opts.n_cpus
if ncpus < 1:
ncpus = cpu_count()
plugin_settings = {
'plugin': 'MultiProc',
'plugin_args': {
'n_procs': ncpus,
'raise_insufficient': False,
'maxtasksperchild': 1,
},
}
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
model = default_path(opts.model, opts.bids_dir, 'model-default_smdl.json')
if opts.model in (None, 'default') and not op.exists(model):
model = 'default'
derivatives = True if not opts.derivatives else opts.derivatives
# Need this when specifying args directly (i.e. neuroscout)
# god bless neuroscout, but let's make it work for others!
if isinstance(derivatives, list) and len(derivatives) == 1:
# WRONG AND EVIL to those who have spaces in their paths... bad bad practice
# TODO - fix neuroscout
derivatives = derivatives[0].split(" ")
if opts.estimator != 'afni':
if opts.error_ts:
raise NotImplementedError(
"Saving the error time series is only implemented for"
" the afni estimator. If this is a feature you want"
f" for {opts.estimator} please let us know on github.")
if opts.derivative_label:
logger.warning('--derivative-label no longer has any effect; '
'set output directory name directly')
os.makedirs(opts.output_dir, exist_ok=True)
fub.write_derivative_description(opts.bids_dir, opts.output_dir,
vars(opts))
work_dir = mkdtemp() if opts.work_dir is None else opts.work_dir
# Go ahead and initialize the layout database
if opts.database_path is None:
database_path = Path(work_dir) / 'dbcache'
reset_database = True
else:
database_path = opts.database_path
reset_database = False
indexer = bids.BIDSLayoutIndexer(ignore=ignore, force_index=force_index)
layout = bids.BIDSLayout(
opts.bids_dir,
derivatives=derivatives,
database_path=database_path,
reset_database=reset_database,
indexer=indexer,
)
subject_list = None
if opts.participant_label is not None:
subject_list = fub.collect_participants(
layout, participant_label=opts.participant_label)
# Build main workflow
logger.log(25, INIT_MSG(version=__version__, subject_list=subject_list))
# TODO: Fix AUTO_MODEL
# if model == 'default':
# models = auto_model(layout)
# else:
# import json
# if op.exists(model):
# model_dict = json.loads(Path(model).read_text())
# models = [model_dict]
model_dict = None
if model == 'default':
retcode = 1
raise NotImplementedError(
"The default model has not been implemented yet.")
else:
import json
if op.exists(model):
model_dict = json.loads(Path(model).read_text())
if not model_dict:
raise ValueError(
f'model_dict cannot be empty. Invalid model filepath {model}.')
graph = BIDSStatsModelsGraph(layout, model_dict)
fitlins_wf = init_fitlins_wf(
database_path,
opts.output_dir,
graph=graph,
analysis_level=opts.analysis_level,
model=model,
space=opts.space,
desc=opts.desc_label,
participants=subject_list,
base_dir=work_dir,
smoothing=opts.smoothing,
drop_missing=opts.drop_missing,
drift_model=opts.drift_model,
estimator=opts.estimator,
errorts=opts.error_ts,
)
fitlins_wf.config = deepcopy(config.get_fitlins_config()._sections)
if opts.work_dir:
# dump crashes in working directory (non /tmp)
fitlins_wf.config['execution']['crashdump_dir'] = opts.work_dir
retcode = 0
if not opts.reports_only:
try:
fitlins_wf.run(**plugin_settings)
except Exception as e:
logger.critical(f"FitLins failed: {e}")
raise
run_context = {
'version': __version__,
'command': ' '.join(sys.argv),
'timestamp': time.strftime('%Y-%m-%d %H:%M:%S %z'),
}
selectors = {'desc': opts.desc_label, 'space': opts.space}
if subject_list is not None:
selectors['subject'] = subject_list
graph.load_collections(**selectors)
report_dict = build_report_dict(opts.output_dir, work_dir, graph)
write_full_report(report_dict, run_context, opts.output_dir)
return retcode
def fw_heudiconv_download():
"""Use fw-heudiconv to download BIDS data."""
subjects = [subject_container.label]
sessions = [session_container.label]
# Use manually specified T1 if it exists
if manual_t1 is not None:
anat_input = manual_t1_path
subject_label = subject_container.label.replace('_',
'x').replace(' ', 'x')
session_label = session_container.label.replace('_',
'x').replace(' ', 'x')
prefix = 'sub-{}_ses-{}_'.format(subject_label, session_label)
return True, anat_input, prefix
# Do the download!
bids_root.parent.mkdir(parents=True, exist_ok=True)
downloads = export.gather_bids(fw, project_label, subjects, sessions)
export.download_bids(fw,
downloads,
str(bids_dir.resolve()),
dry_run=False,
folders_to_download=['anat'])
bids.config.set_option('extension_initial_dot', True) # suppress warning
layout = bids.BIDSLayout(bids_root)
filters = {}
if bids_filter_path:
logger.info("Using {} as BIDS filter.".format(bids_filter_path))
with open(bids_filter_path) as f:
data = json.load(f)
try:
filters = data["t1w"]
except KeyError as ke:
print(ke)
logger.warning("Could not find 't1w' field in json.")
logger.info("BIDS filter file not formatted correctly.")
return False
else:
if bids_sub:
filters["subject"] = [bids_sub]
else:
filters["subject"] = subjects
if bids_ses:
filters["session"] = [bids_ses]
else:
filters["session"] = sessions
if bids_acq:
filters["acquisition"] = bids_acq
if bids_run:
filters["run"] = bids_run
anat_list = layout.get(return_type='file',
suffix='T1w',
extension=['.nii', '.nii.gz'],
**filters)
# if there are multiple files or no files, error out
# otherwise just use the one
if len(anat_list) > 1:
logger.warning(
"Multiple anatomical files found in %s. If you want to process multiple images, use the longitudinal gear.",
bids_root)
return False
elif not len(anat_list) or len(anat_list) == 0:
logger.warning("No anatomical files found in %s", bids_root)
return False
else:
anat_input = anat_list[0]
logger.info("Using {} as input anatomical image.".format(anat_input))
# Generate prefix from bids layout
basename = os.path.basename(anat_input).split('.')[0]
prefix = basename.replace('_T1w', '') + '_'
return True, anat_input, prefix
