def _run_interface(self, runtime): specs = self.inputs.template_spec if isdefined(self.inputs.resolution): specs['resolution'] = self.inputs.resolution if isdefined(self.inputs.atlas): specs['atlas'] = self.inputs.atlas if isdefined(self.inputs.cohort): specs['cohort'] = self.inputs.cohort name = self.inputs.template.strip(":").split(":", 1) if len(name) > 1: specs.update({ k: v for modifier in name[1].split(":") for k, v in [tuple(modifier.split("-"))] if k not in specs }) self._results['t1w_file'] = tf.get(name[0], desc=None, suffix='T1w', **specs) self._results['brain_mask'] = tf.get(name[0], desc='brain', suffix='mask', **specs) return runtime
def main(): args = parse_args() print("\n### Running fmriprep-slurm\n") print(vars(args)) print("\n# Loading pyBIDS database (it might take few hours for a big dataset)...\n") sing_bids_path = os.path.join( SINGULARITY_DATA_PATH, os.path.basename(args.bids_path)) layout = bids.BIDSLayout( sing_bids_path, reset_database=args.force_reindex, ignore=( "code", "stimuli", "sourcedata", "models", re.compile(r"^\."), ) + load_bidsignore(sing_bids_path), ) job_path = os.path.join(SINGULARITY_OUTPUT_PATH, SLURM_JOB_DIR) if not os.path.exists(job_path): os.mkdir(job_path) print("\n# Prefectch templateflow templates ...\n") # prefectch templateflow templates os.environ["TEMPLATEFLOW_HOME"] = TEMPLATEFLOW_HOME tf_api.get(args.output_spaces + ["OASIS30ANTs", "fsLR", "fsaverage"]) print("\n# Processing slurm files into {}\n".format( os.path.join(args.output_path, SLURM_JOB_DIR))) for job_file in run_fmriprep(layout, args): if args.submit: submit_slurm_job(job_file)
def _run_interface(self, runtime): specs = self.inputs.template_spec if isdefined(self.inputs.resolution): specs["resolution"] = self.inputs.resolution if isdefined(self.inputs.atlas): specs["atlas"] = self.inputs.atlas if isdefined(self.inputs.cohort): specs["cohort"] = self.inputs.cohort name = self.inputs.template.strip(":").split(":", 1) if len(name) > 1: specs.update( { k: v for modifier in name[1].split(":") for k, v in [tuple(modifier.split("-"))] if k not in specs } ) self._results["t1w_file"] = tf.get(name[0], desc=None, suffix="T1w", **specs) self._results["brain_mask"] = ( tf.get(name[0], desc="brain", suffix="mask", **specs) or tf.get(name[0], label="brain", suffix="mask", **specs) ) return runtime
def _fetch_fslr_templates(hemi, den): """Fetch the necessary templates for fsaverage to fsLR transform""" import templateflow.api as tf fsaverage_sphere = str( tf.get('fsLR', space='fsaverage', suffix='sphere', hemi=hemi, density='164k')) fslr_sphere = str( tf.get('fsLR', space=None, suffix='sphere', hemi=hemi, density=den)) fsaverage_midthick = str( tf.get('fsLR', space='fsaverage', suffix='midthickness', hemi=hemi, density='164k')) fslr_midthick = str( tf.get('fsLR', space=None, suffix='midthickness', hemi=hemi, density=den)) return fsaverage_sphere, fslr_sphere, fsaverage_midthick, fslr_midthick
def _get_cifti_data(surface, volume, subjects_dir=None, density=None): """ Fetch surface and volumetric label files for CIFTI creation. Parameters ---------- surface : str Target surface space volume : str Target volume space subjects_dir : str, optional Path to FreeSurfer subjects directory (required `fsaverage5`/`fsaverage6` surfaces) density : str, optional Surface density (required for `fsLR` surfaces) Returns ------- annotation_files : list Surface annotation files to allow removal of medial wall label_file : str Volumetric label file of subcortical structures Examples -------- >>> annots, label = _get_cifti_data('fsLR', 'MNI152NLin6Asym', density='32k') >>> annots # doctest: +ELLIPSIS ['.../tpl-fsLR_hemi-L_den-32k_desc-nomedialwall_dparc.label.gii', \ '.../tpl-fsLR_hemi-R_den-32k_desc-nomedialwall_dparc.label.gii'] >>> label # doctest: +ELLIPSIS '.../tpl-MNI152NLin6Asym_res-02_atlas-HCP_dseg.nii.gz' """ if surface not in CIFTI_SURFACES or volume not in CIFTI_VOLUMES: raise NotImplementedError( "Variant (surface: {0}, volume: {1}) is not supported".format(surface, volume) ) tpl_kwargs = {'suffix': 'dseg'} # fMRIPrep grayordinates if volume == "MNI152NLin2009cAsym": tpl_kwargs.update({'resolution': '2', 'desc': 'DKT31'}) annotation_files = sorted(glob(os.path.join( subjects_dir, surface, 'label', '*h.aparc.annot' ))) # HCP grayordinates elif volume == 'MNI152NLin6Asym': # templateflow specific resolutions (2mm, 1.6mm) res = {'32k': '2', '59k': '6'}[density] tpl_kwargs.update({'atlas': 'HCP', 'resolution': res}) annotation_files = [ str(f) for f in tf.get('fsLR', density=density, desc='nomedialwall', suffix='dparc') ] if len(annotation_files) != 2: raise IOError("Invalid number of surface annotation files") label_file = str(tf.get(volume, **tpl_kwargs)) return annotation_files, label_file
def make_parcellation(data_path, atlas, template='MNI152NLin2009cAsym', atlas_desc=None, resolution=2, parc_params=None, return_meta=False): """ Performs a parcellation which reduces voxel space to regions of interest (brain data). Parameters ---------- data_path : str Path to .nii image. atlas : str Specify which atlas you want to use (see github.com/templateflow/) template : str What space you data is in. If fmriprep, leave as MNI152NLin2009cAsym. atlas_desc : str Specify which description of atlas. resolution : int Resolution of atlas. Can be 1 or 2. parc_params : dict **kwargs for nilearn functions. return_meta : bool If true, tries to return any meta-information that exists about parcellation. Returns ------- data : array Data after the parcellation. NOTE ---- These functions make use of nilearn. Please cite templateflow and nilearn if used in a publicaiton. """ if not parc_params: parc_params = {} tf_get_params = { 'template': template, 'resolution': resolution, 'atlas': atlas } if atlas_desc is not None: tf_get_params['desc'] = atlas_desc file = tf.get(**tf_get_params, extensions='nii.gz') if isinstance(file, list): raise ValueError('More than one template file found. Specify the type of file you need (often atlas_desc). Run: templateflow.api.TF_LAYOUT.get_descs(atlas=' + atlas + ') to see available desc for atlas') region = NiftiLabelsMasker(str(file), **parc_params) data = region.fit_transform(data_path) if return_meta: meta_info = tf.get(template=template, atlas=atlas, desc=atlas_desc, extensions='tsv') meta_info = load_tabular_file(str(meta_info)) return data, meta_info else: return data
def fetch_fsaverage(): """ Expected templates: tpl-fsaverage/tpl-fsaverage_hemi-L_den-164k_desc-std_sphere.surf.gii tpl-fsaverage/tpl-fsaverage_hemi-R_den-164k_desc-std_sphere.surf.gii tpl-fsaverage/tpl-fsaverage_hemi-L_den-164k_desc-vaavg_midthickness.shape.gii tpl-fsaverage/tpl-fsaverage_hemi-R_den-164k_desc-vaavg_midthickness.shape.gii """ template = 'fsaverage' tf.get(template, density='164k', desc='std', suffix='sphere') tf.get(template, density='164k', desc='vaavg', suffix='midthickness')
def fetch_fsLR(): """ Expected templates: tpl-fsLR/tpl-fsLR_hemi-L_den-32k_desc-nomedialwall_dparc.label.gii tpl-fsLR/tpl-fsLR_hemi-L_den-32k_desc-vaavg_midthickness.shape.gii tpl-fsLR/tpl-fsLR_hemi-L_den-32k_sphere.surf.gii tpl-fsLR/tpl-fsLR_hemi-R_den-32k_desc-nomedialwall_dparc.label.gii tpl-fsLR/tpl-fsLR_hemi-R_den-32k_desc-vaavg_midthickness.shape.gii tpl-fsLR/tpl-fsLR_hemi-R_den-32k_sphere.surf.gii tpl-fsLR/tpl-fsLR_space-fsaverage_hemi-L_den-32k_sphere.surf.gii tpl-fsLR/tpl-fsLR_space-fsaverage_hemi-R_den-32k_sphere.surf.gii """ tf.get('fsLR', density='32k')
def test_transformer_nii(tmp_path): os.chdir(str(tmp_path)) api_args = dict(template="MNI152NLin2009cAsym", resolution=2) ref_mask_file = api.get(**api_args, desc="brain", suffix="mask") ref_mask_img = nib.load(ref_mask_file) ref_mask = ref_mask_img.get_fdata() > 0 n_voxels = np.count_nonzero(ref_mask) n_volumes = 10 test_array = np.random.rand(n_voxels, n_volumes) test_img_data = np.zeros((*ref_mask_img.shape, n_volumes), dtype=float) test_img_data[ref_mask, :] = test_array img = new_img_like(ref_mask_img, test_img_data, copy_header=True) img.header.set_data_dtype(np.float64) test_file = "img.nii.gz" nib.save(img, test_file) tf = Transformer() tf.inputs.mask = ref_mask_file array = tf._load(test_file) out_file = tf._dump(array) img_data = nib.load(out_file).get_fdata() assert np.allclose(test_img_data, img_data)
def _run_interface(self, runtime): specs = self.inputs.template_spec if isdefined(self.inputs.resolution): specs['resolution'] = self.inputs.resolution if isdefined(self.inputs.atlas): specs['atlas'] = self.inputs.atlas self._results['t1w_file'] = tf.get(self.inputs.template, desc=None, suffix='T1w', **specs) self._results['brain_mask'] = tf.get(self.inputs.template, desc='brain', suffix='mask', **specs) return runtime
def get_surface_meshes(density, surface_type): import templateflow.api as tf lh, rh = tf.get("fsLR", density=density, suffix=surface_type, extension=[".surf.gii"]) return str(lh), str(rh)
def test_nifti_input(): nifti = nib.load( tflow.get("MNI152Lin", resolution="02", desc="brain", suffix="mask")) edg = mesh_edges(nifti) assert edg.shape[1] == 2 assert np.amax(edg) <= nifti.get_data().sum() - 1
def get_atlas_path(info_dict, args): info_dict['atlas_desc'] = f'{args.nparcels}Parcels{args.nnetworks}Networks' info_dict['atlas_path'] = str( tflow.get(info_dict['space'], desc=info_dict['atlas_desc'], resolution=args.atlas_resolution, atlas=args.atlas)) return(info_dict)
def _templateflow_setup(config_file=None, debug=False): if not debug: sys.excepthook = exception_handler logging.basicConfig(level=logging.INFO) else: logging.basicConfig(level=logging.DEBUG) config = ClpipeConfigParser() config.config_updater(config_file) templateflow_path = config.config["FMRIPrepOptions"]["TemplateFlowPath"] logging.info("Setting TemplateFlow storage path to " + templateflow_path) os.system("export TEMPLATEFLOW_HOME=" + templateflow_path) logging.info( "Downloading requested templates " + " ".join(config.config['FMRIPrepOptions']["TemplateFlowTemplates"])) api.get(config.config['FMRIPrepOptions']["TemplateFlowTemplates"])
def fetch_MNI2009(): """ Expected templates: tpl-MNI152NLin2009cAsym/tpl-MNI152NLin2009cAsym_res-01_T1w.nii.gz tpl-MNI152NLin2009cAsym/tpl-MNI152NLin2009cAsym_res-02_T1w.nii.gz tpl-MNI152NLin2009cAsym/tpl-MNI152NLin2009cAsym_res-01_desc-brain_mask.nii.gz tpl-MNI152NLin2009cAsym/tpl-MNI152NLin2009cAsym_res-02_desc-brain_mask.nii.gz tpl-MNI152NLin2009cAsym/tpl-MNI152NLin2009cAsym_res-01_desc-carpet_dseg.nii.gz tpl-MNI152NLin2009cAsym/tpl-MNI152NLin2009cAsym_res-02_desc-fMRIPrep_boldref.nii.gz tpl-MNI152NLin2009cAsym/tpl-MNI152NLin2009cAsym_res-01_label-brain_probseg.nii.gz """ template = 'MNI152NLin2009cAsym' tf.get(template, resolution=(1, 2), desc=None, suffix='T1w') tf.get(template, resolution=(1, 2), desc='brain', suffix='mask') tf.get(template, resolution=1, atlas=None, desc='carpet', suffix='dseg') tf.get(template, resolution=2, desc='fMRIPrep', suffix='boldref') tf.get(template, resolution=1, label="brain", suffix="probseg")
def from_templateflow(self, in_prefix, **kwargs): in_atlas_path = api.get( self.template, resolution=self.resolution, **kwargs ) in_atlas_img = nib.load(in_atlas_path) in_atlas = np.asanyarray(in_atlas_img.dataobj, dtype=np.uint16) in_labels_path = str(first( filter( lambda f: f.suffix == ".tsv", api.get(self.template, **kwargs) ) )) in_labels_df = pd.read_table(in_labels_path, sep=r"\s+", index_col=0) assert isinstance(in_labels_df, pd.DataFrame) in_labels = in_labels_df["name"] self.accumulate(in_prefix, in_labels, in_atlas)
def __init__(self, template="MNI152NLin2009cAsym", resolution=2): self.template = template self.resolution = resolution self.fixed_img_path = api.get( template, resolution=resolution, suffix="T1w", desc="brain" ) self.fixed_img = nib.load(self.fixed_img_path) self.masks = pd.Series([], dtype=object) self.labels = pd.Series([], dtype=object)
def test_volumetric_input(): mask_image = nib.load( tflow.get("MNI152Lin", resolution="02", desc="brain", suffix="mask")) n_voxels = (mask_image.get_fdata() != 0).sum() n_subjects = 3 data = np.random.rand(n_subjects, n_voxels) model = FixedEffect(1) contrast = np.ones(3) slm = SLM(model, contrast, surf=mask_image) slm.fit(data)
def read_mni_template(resolution=1, mask=True): """ Reads the MNI T2w template Parameters ---------- resolution : int, optional. Either 1 or 2, the resolution in mm of the voxels. Default: 1. mask : bool, optional Whether to mask the data with a brain-mask before returning the image. Default : True Returns ------- nib.Nifti1Image class instance containing masked or unmasked T2 template. """ template_img = nib.load( str( tflow.get('MNI152NLin2009cAsym', desc=None, resolution=resolution, suffix='T2w', extension='nii.gz'))) if not mask: return template_img else: mask_img = nib.load( str( tflow.get('MNI152NLin2009cAsym', resolution=resolution, desc='brain', suffix='mask'))) template_data = template_img.get_fdata() mask_data = mask_img.get_fdata() out_data = template_data * mask_data return nib.Nifti1Image(out_data, template_img.affine)
def fetch_OASIS(): """ Expected templates: tpl-OASIS30ANTs/tpl-OASIS30ANTs_res-01_T1w.nii.gz tpl-OASIS30ANTs/tpl-OASIS30ANTs_res-01_label-brain_probseg.nii.gz tpl-OASIS30ANTs/tpl-OASIS30ANTs_res-01_desc-brain_mask.nii.gz tpl-OASIS30ANTs/tpl-OASIS30ANTs_res-01_desc-BrainCerebellumExtraction_mask.nii.gz """ template = "OASIS30ANTs" tf.get(template, resolution=1, desc=None, label=None, suffix='T1w') tf.get(template, resolution=1, label='brain', suffix='probseg') tf.get(template, resolution=1, label='brain', suffix='mask') tf.get(template, resolution=1, desc='BrainCerebellumExtraction', suffix='mask')
def _plot_template(ax, style='filled', template='MNI152NLin2009cAsym', templatecolor='lightgray', alpha=0.2, voxsize=None, azim=0, elev=0, surface_detection=None, surface_resolution=2, edgethreshold=0.8, hemisphere='both'): if isinstance(template, str): if not os.path.exists(template): if surface_detection is None and style == 'surface': surface_detection = _get_surface_level_for_template(template) tf_kwargs = {} # Add kwargs to specify specific templates if 'MNI152' in template or 'OASIS' in template: tf_kwargs = { 'suffix': 'T1w', 'resolution': 1, } if 'WHS' in template: tf_kwargs = { 'resolution': 1, } template = tf.get(template=template, desc='brain', extension='.nii.gz', **tf_kwargs) # If multiple templates still remain, take the first # This may lead to suboptimal performence for some templates if isinstance(template, list): template = template[0] img = nib.load(template) elif isinstance(template, (nib.Nifti1Image, nib.Nifti2Image)): img = template if voxsize is not None: img = resample_to_output(img, [voxsize] * 3) data = img.get_fdata() data = _select_single_hemisphere_template(data, hemisphere) if style == 'filled': _plot_template_style_filled(ax, data, alpha, templatecolor) elif style == 'cloudy': _plot_template_style_cloudy(ax, data, azim, elev, edgethreshold, templatecolor, alpha) elif style == 'surface': _plot_template_style_surface(ax, data, alpha, template, templatecolor, surface_resolution, surface_detection) return img.affine
def main(): args = parse_args() pybids_cache_path = os.path.join(args.bids_path, PYBIDS_CACHE_PATH) layout = bids.BIDSLayout( args.bids_path, database_path=pybids_cache_path, reset_database=args.force_reindex, ignore=( "code", "stimuli", "sourcedata", "models", re.compile(r"^\."), ) + load_bidsignore(args.bids_path), ) job_path = os.path.join(layout.root, SLURM_JOB_DIR) if not os.path.exists(job_path): os.mkdir(job_path) # add .slurm to .gitignore with open(os.path.join(layout.root, ".gitignore"), "a+") as f: f.seek(0) if not any([SLURM_JOB_DIR in l for l in f.readlines()]): f.write(f"{SLURM_JOB_DIR}\n") # prefectch templateflow templates os.environ["TEMPLATEFLOW_HOME"] = TEMPLATEFLOW_HOME import templateflow.api as tf_api tf_api.get(OUTPUT_TEMPLATES + ["OASIS30ANTs", "fsLR", "fsaverage"]) for job_file in run_fmriprep(layout, args, args.preproc): if not args.no_submit: submit_slurm_job(job_file)
def get_imgs(): root = tk.Tk() root.withdraw() bg_img = load_img( str(tflow.get('MNI152NLin6Asym', desc=None, resolution=1, suffix='T1w'))) stats_img = load_img( fd.askopenfilename(title='select stats image', filetypes=[('nifti files', '*.nii *.gz')])) seed_img = None if args.seed: seed_img = load_img( fd.askopenfilename(title='select seed image', filetypes=[('nifti files', '*.nii *.gz')])) output_path = fd.asksaveasfilename(title='name output file') return bg_img, stats_img, seed_img, output_path
def fetch_UNCInfant(): """ Expected templates: tpl-UNCInfant/cohort-1/tpl-UNCInfant_cohort-1_T1w.nii.gz tpl-UNCInfant/cohort-1/tpl-UNCInfant_cohort-1_label-brain_probseg.nii.gz tpl-UNCInfant/cohort-1/tpl-UNCInfant_cohort-1_label-brain_mask.nii.gz tpl-UNCInfant/cohort-1/tpl-UNCInfant_cohort-1_label-BrainCerebellumExtraction_mask.nii.gz """ template = "UNCInfant" tf.get(template, cohort=1, desc=None, suffix='T1w') tf.get(template, cohort=1, label='brain', suffix='probseg') tf.get(template, cohort=1, label='brain', suffix='mask') tf.get(template, cohort=1, label='BrainCerebellumExtraction', suffix='mask')
def fetch_MNI6(): """ Expected templates: tpl-MNI152NLin6Asym/tpl-MNI152NLin6Asym_res-01_T1w.nii.gz tpl-MNI152NLin6Asym/tpl-MNI152NLin6Asym_res-02_T1w.nii.gz tpl-MNI152NLin6Asym/tpl-MNI152NLin6Asym_res-01_desc-brain_mask.nii.gz tpl-MNI152NLin6Asym/tpl-MNI152NLin6Asym_res-02_desc-brain_mask.nii.gz tpl-MNI152NLin6Asym/tpl-MNI152NLin6Asym_res-02_atlas-HCP_dseg.nii.gz """ template = 'MNI152NLin6Asym' tf.get(template, resolution=(1, 2), desc=None, suffix='T1w') tf.get(template, resolution=(1, 2), desc='brain', suffix='mask') # CIFTI tf.get(template, resolution=2, atlas='HCP', suffix='dseg')
def medial_wall_to_nan(in_file, subjects_dir, den=None, newpath=None): """Convert values on medial wall to NaNs.""" import os import nibabel as nb import numpy as np import templateflow.api as tf fn = os.path.basename(in_file) target_subject = in_file.split(".")[1] if not target_subject.startswith('fs'): return in_file func = nb.load(in_file) if target_subject.startswith('fsaverage'): cortex = nb.freesurfer.read_label( os.path.join(subjects_dir, target_subject, 'label', '{}.cortex.label'.format(fn[:2]))) medial = np.delete(np.arange(len(func.darrays[0].data)), cortex) elif target_subject == 'fslr' and den is not None: hemi = fn[0].upper() label_file = str( tf.get('fsLR', hemi=hemi, desc='nomedialwall', density=den, suffix='dparc')) label = nb.load(label_file) medial = np.invert(label.darrays[0].data.astype(bool)) else: return in_file for darray in func.darrays: darray.data[medial] = np.nan out_file = os.path.join(newpath or os.getcwd(), fn) func.to_filename(out_file) return out_file
def test_zscore(tmp_path): os.chdir(str(tmp_path)) api_args = dict(template="MNI152NLin2009cAsym", resolution=2) ref_mask_file = api.get(**api_args, desc="brain", suffix="mask") ref_mask_img = nib.load(ref_mask_file) ref_mask = ref_mask_img.get_fdata() > 0 n_voxels = np.count_nonzero(ref_mask) test_data = np.random.rand(n_voxels) assert not isclose(np.mean(test_data), 0, abs_tol=abs_tol) assert not isclose(np.std(test_data), 1, abs_tol=abs_tol) test_img_data = np.zeros(ref_mask_img.shape, dtype=float) test_img_data[ref_mask] = test_data img = new_img_like(ref_mask_img, test_img_data, copy_header=True) assert isinstance(img.header, nib.Nifti1Header) img.header.set_data_dtype(np.float64) test_file = "img.nii.gz" nib.save(img, test_file) instance = ZScore() instance.inputs.in_file = test_file instance.inputs.mask = ref_mask_file result = instance.run() assert result.outputs is not None out_img = nib.load(result.outputs.out_file) out_img_data = out_img.get_fdata() out_data = out_img_data[ref_mask] assert isclose(np.mean(out_data), 0, abs_tol=abs_tol) assert isclose(np.std(out_data), 1, abs_tol=abs_tol)
def main(): """Entry point""" from nipype import logging as nlogging from multiprocessing import set_start_method, Process, Manager from ..utils.bids import write_derivative_description, validate_input_dir set_start_method('forkserver') warnings.showwarning = _warn_redirect opts = get_parser().parse_args() exec_env = os.name # special variable set in the container if os.getenv('IS_DOCKER_8395080871'): exec_env = 'singularity' cgroup = Path('/proc/1/cgroup') if cgroup.exists() and 'docker' in cgroup.read_text(): exec_env = 'docker' if os.getenv('DOCKER_VERSION_8395080871'): exec_env = 'fmriprep-docker' sentry_sdk = None if not opts.notrack: import sentry_sdk from ..utils.sentry import sentry_setup sentry_setup(opts, exec_env) if opts.debug: print('WARNING: Option --debug is deprecated and has no effect', file=sys.stderr) # Validate inputs if not opts.skip_bids_validation: print( "Making sure the input data is BIDS compliant (warnings can be ignored in most " "cases).") validate_input_dir(exec_env, opts.bids_dir, opts.participant_label) # FreeSurfer license default_license = str(Path(os.getenv('FREESURFER_HOME')) / 'license.txt') # Precedence: --fs-license-file, $FS_LICENSE, default_license license_file = opts.fs_license_file or Path( os.getenv('FS_LICENSE', default_license)) if not license_file.exists(): raise RuntimeError("""\ ERROR: a valid license file is required for FreeSurfer to run. fMRIPrep looked for an existing \ license file at several paths, in this order: 1) command line argument ``--fs-license-file``; \ 2) ``$FS_LICENSE`` environment variable; and 3) the ``$FREESURFER_HOME/license.txt`` path. Get it \ (for free) by registering at https://surfer.nmr.mgh.harvard.edu/registration.html""" ) os.environ['FS_LICENSE'] = str(license_file.resolve()) # Retrieve logging level log_level = int(max(25 - 5 * opts.verbose_count, logging.DEBUG)) # Set logging logger.setLevel(log_level) nlogging.getLogger('nipype.workflow').setLevel(log_level) nlogging.getLogger('nipype.interface').setLevel(log_level) nlogging.getLogger('nipype.utils').setLevel(log_level) # Call build_workflow(opts, retval) with Manager() as mgr: retval = mgr.dict() p = Process(target=build_workflow, args=(opts, retval)) p.start() p.join() retcode = p.exitcode or retval.get('return_code', 0) bids_dir = Path(retval.get('bids_dir')) output_dir = Path(retval.get('output_dir')) work_dir = Path(retval.get('work_dir')) plugin_settings = retval.get('plugin_settings', None) subject_list = retval.get('subject_list', None) fmriprep_wf = retval.get('workflow', None) run_uuid = retval.get('run_uuid', None) if opts.reports_only: sys.exit(int(retcode > 0)) if opts.boilerplate: sys.exit(int(retcode > 0)) if fmriprep_wf and opts.write_graph: fmriprep_wf.write_graph(graph2use="colored", format='svg', simple_form=True) retcode = retcode or int(fmriprep_wf is None) if retcode != 0: sys.exit(retcode) # Check workflow for missing commands missing = check_deps(fmriprep_wf) if missing: print("Cannot run fMRIPrep. Missing dependencies:", file=sys.stderr) for iface, cmd in missing: print("\t{} (Interface: {})".format(cmd, iface)) sys.exit(2) # Clean up master process before running workflow, which may create forks gc.collect() # Sentry tracking if not opts.notrack: from ..utils.sentry import start_ping start_ping(run_uuid, len(subject_list)) errno = 1 # Default is error exit unless otherwise set try: fmriprep_wf.run(**plugin_settings) except Exception as e: if not opts.notrack: from ..utils.sentry import process_crashfile crashfolders = [ output_dir / 'fmriprep' / 'sub-{}'.format(s) / 'log' / run_uuid for s in subject_list ] for crashfolder in crashfolders: for crashfile in crashfolder.glob('crash*.*'): process_crashfile(crashfile) if "Workflow did not execute cleanly" not in str(e): sentry_sdk.capture_exception(e) logger.critical('fMRIPrep failed: %s', e) raise else: if opts.run_reconall: from templateflow import api from niworkflows.utils.misc import _copy_any dseg_tsv = str( api.get('fsaverage', suffix='dseg', extension=['.tsv'])) _copy_any(dseg_tsv, str(output_dir / 'fmriprep' / 'desc-aseg_dseg.tsv')) _copy_any(dseg_tsv, str(output_dir / 'fmriprep' / 'desc-aparcaseg_dseg.tsv')) errno = 0 logger.log(25, 'fMRIPrep finished without errors') if not opts.notrack: sentry_sdk.capture_message('fMRIPrep finished without errors', level='info') finally: from niworkflows.reports import generate_reports from subprocess import check_call, CalledProcessError, TimeoutExpired from pkg_resources import resource_filename as pkgrf from shutil import copyfile citation_files = { ext: output_dir / 'fmriprep' / 'logs' / ('CITATION.%s' % ext) for ext in ('bib', 'tex', 'md', 'html') } if citation_files['md'].exists(): # Generate HTML file resolving citations cmd = [ 'pandoc', '-s', '--bibliography', pkgrf('fmriprep', 'data/boilerplate.bib'), '--filter', 'pandoc-citeproc', '--metadata', 'pagetitle="fMRIPrep citation boilerplate"', str(citation_files['md']), '-o', str(citation_files['html']) ] logger.info( 'Generating an HTML version of the citation boilerplate...') try: check_call(cmd, timeout=10) except (FileNotFoundError, CalledProcessError, TimeoutExpired): logger.warning('Could not generate CITATION.html file:\n%s', ' '.join(cmd)) # Generate LaTex file resolving citations cmd = [ 'pandoc', '-s', '--bibliography', pkgrf('fmriprep', 'data/boilerplate.bib'), '--natbib', str(citation_files['md']), '-o', str(citation_files['tex']) ] logger.info( 'Generating a LaTeX version of the citation boilerplate...') try: check_call(cmd, timeout=10) except (FileNotFoundError, CalledProcessError, TimeoutExpired): logger.warning('Could not generate CITATION.tex file:\n%s', ' '.join(cmd)) else: copyfile(pkgrf('fmriprep', 'data/boilerplate.bib'), citation_files['bib']) else: logger.warning( 'fMRIPrep could not find the markdown version of ' 'the citation boilerplate (%s). HTML and LaTeX versions' ' of it will not be available', citation_files['md']) # Generate reports phase failed_reports = generate_reports(subject_list, output_dir, work_dir, run_uuid, packagename='fmriprep') write_derivative_description(bids_dir, output_dir / 'fmriprep') if failed_reports and not opts.notrack: sentry_sdk.capture_message( 'Report generation failed for %d subjects' % failed_reports, level='error') sys.exit(int((errno + failed_reports) > 0))
def main(): """Entry point.""" import os import sys import gc from multiprocessing import Process, Manager from .parser import parse_args from ..utils.bids import write_derivative_description parse_args() popylar = None if not config.execution.notrack: import popylar from ..__about__ import __ga_id__ config.loggers.cli.info( "Your usage of dmriprep is being recorded using popylar (https://popylar.github.io/). ", # noqa "For details, see https://nipreps.github.io/dmriprep/usage.html. ", "To opt out, call dmriprep with a `--notrack` flag") popylar.track_event(__ga_id__, 'run', 'cli_run') # CRITICAL Save the config to a file. This is necessary because the execution graph # is built as a separate process to keep the memory footprint low. The most # straightforward way to communicate with the child process is via the filesystem. config_file = config.execution.work_dir / '.dmriprep.toml' config.to_filename(config_file) # CRITICAL Call build_workflow(config_file, retval) in a subprocess. # Because Python on Linux does not ever free virtual memory (VM), running the # workflow construction jailed within a process preempts excessive VM buildup. with Manager() as mgr: from .workflow import build_workflow retval = mgr.dict() p = Process(target=build_workflow, args=(str(config_file), retval)) p.start() p.join() retcode = p.exitcode or retval.get('return_code', 0) dmriprep_wf = retval.get('workflow', None) # CRITICAL Load the config from the file. This is necessary because the ``build_workflow`` # function executed constrained in a process may change the config (and thus the global # state of dMRIPrep). config.load(config_file) if config.execution.reports_only: sys.exit(int(retcode > 0)) if dmriprep_wf and config.execution.write_graph: dmriprep_wf.write_graph(graph2use="colored", format='svg', simple_form=True) retcode = retcode or (dmriprep_wf is None) * os.EX_SOFTWARE if retcode != 0: sys.exit(retcode) # Generate boilerplate with Manager() as mgr: from .workflow import build_boilerplate p = Process(target=build_boilerplate, args=(str(config_file), dmriprep_wf)) p.start() p.join() if config.execution.boilerplate_only: sys.exit(int(retcode > 0)) # Clean up master process before running workflow, which may create forks gc.collect() if popylar is not None: popylar.track_event(__ga_id__, 'run', 'started') config.loggers.workflow.log( 15, '\n'.join(['dMRIPrep config:'] + ['\t\t%s' % s for s in config.dumps().splitlines()])) config.loggers.workflow.log(25, 'dMRIPrep started!') errno = 1 # Default is error exit unless otherwise set try: dmriprep_wf.run(**config.nipype.get_plugin()) except Exception as e: if not config.execution.notrack: popylar.track_event(__ga_id__, 'run', 'error') config.loggers.workflow.critical('dMRIPrep failed: %s', e) raise else: config.loggers.workflow.log(25, 'dMRIPrep finished successfully!') # Bother users with the boilerplate only iff the workflow went okay. if (config.execution.output_dir / 'dmriprep' / 'logs' / 'CITATION.md').exists(): config.loggers.workflow.log( 25, 'Works derived from this dMRIPrep execution should ' 'include the following boilerplate:\n\n%s', (config.execution.output_dir / 'dmriprep' / 'logs' / 'CITATION.md').read_text()) if config.workflow.run_reconall: from templateflow import api from niworkflows.utils.misc import _copy_any dseg_tsv = str( api.get('fsaverage', suffix='dseg', extension=['.tsv'])) _copy_any( dseg_tsv, str(config.execution.output_dir / 'dmriprep' / 'desc-aseg_dseg.tsv')) _copy_any( dseg_tsv, str(config.execution.output_dir / 'dmriprep' / 'desc-aparcaseg_dseg.tsv')) errno = 0 finally: from niworkflows.reports import generate_reports from pkg_resources import resource_filename as pkgrf # Generate reports phase failed_reports = generate_reports(config.execution.participant_label, config.execution.output_dir, config.execution.work_dir, config.execution.run_uuid, config=pkgrf( 'dmriprep', 'config/reports-spec.yml'), packagename='dmriprep') write_derivative_description(config.execution.bids_dir, config.execution.output_dir / 'dmriprep') if failed_reports and not config.execution.notrack: popylar.track_event(__ga_id__, 'run', 'reporting_error') sys.exit(int((errno + failed_reports) > 0))
def build_opts(opts): """Trigger a new process that builds the workflow graph, based on the input options.""" import os from pathlib import Path import logging import sys import gc import warnings from multiprocessing import set_start_method, Process, Manager from nipype import logging as nlogging from niworkflows.utils.misc import check_valid_fs_license set_start_method('forkserver') logging.addLevelName( 25, 'IMPORTANT') # Add a new level between INFO and WARNING logging.addLevelName(15, 'VERBOSE') # Add a new level between INFO and DEBUG logger = logging.getLogger('cli') def _warn_redirect(message, category, filename, lineno, file=None, line=None): logger.warning('Captured warning (%s): %s', category, message) warnings.showwarning = _warn_redirect # Precedence: --fs-license-file, $FS_LICENSE, default_license if opts.fs_license_file is not None: os.environ["FS_LICENSE"] = os.path.abspath(opts.fs_license_file) if not check_valid_fs_license(): raise RuntimeError( 'ERROR: a valid license file is required for FreeSurfer to run. ' 'sMRIPrep looked for an existing license file at several paths, in this ' 'order: 1) command line argument ``--fs-license-file``; 2) ``$FS_LICENSE`` ' 'environment variable; and 3) the ``$FREESURFER_HOME/license.txt`` path. ' 'Get it (for free) by registering at https://' 'surfer.nmr.mgh.harvard.edu/registration.html') # Retrieve logging level log_level = int(max(25 - 5 * opts.verbose_count, logging.DEBUG)) # Set logging logger.setLevel(log_level) nlogging.getLogger('nipype.workflow').setLevel(log_level) nlogging.getLogger('nipype.interface').setLevel(log_level) nlogging.getLogger('nipype.utils').setLevel(log_level) errno = 0 # Call build_workflow(opts, retval) with Manager() as mgr: retval = mgr.dict() p = Process(target=build_workflow, args=(opts, retval)) p.start() p.join() if p.exitcode != 0: sys.exit(p.exitcode) smriprep_wf = retval['workflow'] plugin_settings = retval['plugin_settings'] bids_dir = retval['bids_dir'] output_dir = retval['output_dir'] subject_list = retval['subject_list'] run_uuid = retval['run_uuid'] retcode = retval['return_code'] if smriprep_wf is None: sys.exit(1) if opts.write_graph: smriprep_wf.write_graph(graph2use="colored", format='svg', simple_form=True) if opts.reports_only: sys.exit(int(retcode > 0)) if opts.boilerplate: sys.exit(int(retcode > 0)) # Check workflow for missing commands missing = check_deps(smriprep_wf) if missing: print("Cannot run sMRIPrep. Missing dependencies:") for iface, cmd in missing: print("\t{} (Interface: {})".format(cmd, iface)) sys.exit(2) # Clean up master process before running workflow, which may create forks gc.collect() try: smriprep_wf.run(**plugin_settings) except RuntimeError: errno = 1 else: if opts.run_reconall: from templateflow import api from niworkflows.utils.misc import _copy_any dseg_tsv = str( api.get('fsaverage', suffix='dseg', extension=['.tsv'])) _copy_any( dseg_tsv, str(Path(output_dir) / 'smriprep' / 'desc-aseg_dseg.tsv')) _copy_any( dseg_tsv, str(Path(output_dir) / 'smriprep' / 'desc-aparcaseg_dseg.tsv')) logger.log(25, 'sMRIPrep finished without errors') finally: from niworkflows.reports import generate_reports from ..utils.bids import write_derivative_description logger.log(25, 'Writing reports for participants: %s', ', '.join(subject_list)) # Generate reports phase errno += generate_reports(subject_list, output_dir, run_uuid, packagename='smriprep') write_derivative_description(bids_dir, str(Path(output_dir) / 'smriprep')) sys.exit(int(errno > 0))