def main():
"""Entry point"""
from nipype import logging as nlogging
from multiprocessing import set_start_method, Process, Manager
from ..utils.bids import write_derivative_description, validate_input_dir
set_start_method('forkserver')
warnings.showwarning = _warn_redirect
opts = get_parser().parse_args()
exec_env = os.name
# special variable set in the container
if os.getenv('IS_DOCKER_8395080871'):
exec_env = 'singularity'
cgroup = Path('/proc/1/cgroup')
if cgroup.exists() and 'docker' in cgroup.read_text():
exec_env = 'docker'
if os.getenv('DOCKER_VERSION_8395080871'):
exec_env = 'fmriprep-docker'
sentry_sdk = None
if not opts.notrack:
import sentry_sdk
from ..utils.sentry import sentry_setup
sentry_setup(opts, exec_env)
if opts.debug:
print('WARNING: Option --debug is deprecated and has no effect',
file=sys.stderr)
# Validate inputs
if not opts.skip_bids_validation:
print(
"Making sure the input data is BIDS compliant (warnings can be ignored in most "
"cases).")
validate_input_dir(exec_env, opts.bids_dir, opts.participant_label)
# FreeSurfer license
default_license = str(Path(os.getenv('FREESURFER_HOME')) / 'license.txt')
# Precedence: --fs-license-file, $FS_LICENSE, default_license
license_file = opts.fs_license_file or Path(
os.getenv('FS_LICENSE', default_license))
if not license_file.exists():
raise RuntimeError("""\
ERROR: a valid license file is required for FreeSurfer to run. fMRIPrep looked for an existing \
license file at several paths, in this order: 1) command line argument ``--fs-license-file``; \
2) ``$FS_LICENSE`` environment variable; and 3) the ``$FREESURFER_HOME/license.txt`` path. Get it \
(for free) by registering at https://surfer.nmr.mgh.harvard.edu/registration.html"""
)
os.environ['FS_LICENSE'] = str(license_file.resolve())
# Retrieve logging level
log_level = int(max(25 - 5 * opts.verbose_count, logging.DEBUG))
# Set logging
logger.setLevel(log_level)
nlogging.getLogger('nipype.workflow').setLevel(log_level)
nlogging.getLogger('nipype.interface').setLevel(log_level)
nlogging.getLogger('nipype.utils').setLevel(log_level)
# Call build_workflow(opts, retval)
with Manager() as mgr:
retval = mgr.dict()
p = Process(target=build_workflow, args=(opts, retval))
p.start()
p.join()
retcode = p.exitcode or retval.get('return_code', 0)
bids_dir = Path(retval.get('bids_dir'))
output_dir = Path(retval.get('output_dir'))
work_dir = Path(retval.get('work_dir'))
plugin_settings = retval.get('plugin_settings', None)
subject_list = retval.get('subject_list', None)
fmriprep_wf = retval.get('workflow', None)
run_uuid = retval.get('run_uuid', None)
if opts.reports_only:
sys.exit(int(retcode > 0))
if opts.boilerplate:
sys.exit(int(retcode > 0))
if fmriprep_wf and opts.write_graph:
fmriprep_wf.write_graph(graph2use="colored",
format='svg',
simple_form=True)
retcode = retcode or int(fmriprep_wf is None)
if retcode != 0:
sys.exit(retcode)
# Check workflow for missing commands
missing = check_deps(fmriprep_wf)
if missing:
print("Cannot run fMRIPrep. Missing dependencies:", file=sys.stderr)
for iface, cmd in missing:
print("\t{} (Interface: {})".format(cmd, iface))
sys.exit(2)
# Clean up master process before running workflow, which may create forks
gc.collect()
# Sentry tracking
if not opts.notrack:
from ..utils.sentry import start_ping
start_ping(run_uuid, len(subject_list))
errno = 1 # Default is error exit unless otherwise set
try:
fmriprep_wf.run(**plugin_settings)
except Exception as e:
if not opts.notrack:
from ..utils.sentry import process_crashfile
crashfolders = [
output_dir / 'fmriprep' / 'sub-{}'.format(s) / 'log' / run_uuid
for s in subject_list
]
for crashfolder in crashfolders:
for crashfile in crashfolder.glob('crash*.*'):
process_crashfile(crashfile)
if "Workflow did not execute cleanly" not in str(e):
sentry_sdk.capture_exception(e)
logger.critical('fMRIPrep failed: %s', e)
raise
else:
if opts.run_reconall:
from templateflow import api
from niworkflows.utils.misc import _copy_any
dseg_tsv = str(
api.get('fsaverage', suffix='dseg', extension=['.tsv']))
_copy_any(dseg_tsv,
str(output_dir / 'fmriprep' / 'desc-aseg_dseg.tsv'))
_copy_any(dseg_tsv,
str(output_dir / 'fmriprep' / 'desc-aparcaseg_dseg.tsv'))
errno = 0
logger.log(25, 'fMRIPrep finished without errors')
if not opts.notrack:
sentry_sdk.capture_message('fMRIPrep finished without errors',
level='info')
finally:
from niworkflows.reports import generate_reports
from subprocess import check_call, CalledProcessError, TimeoutExpired
from pkg_resources import resource_filename as pkgrf
from shutil import copyfile
citation_files = {
ext: output_dir / 'fmriprep' / 'logs' / ('CITATION.%s' % ext)
for ext in ('bib', 'tex', 'md', 'html')
}
if citation_files['md'].exists():
# Generate HTML file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('fmriprep',
'data/boilerplate.bib'), '--filter', 'pandoc-citeproc',
'--metadata', 'pagetitle="fMRIPrep citation boilerplate"',
str(citation_files['md']), '-o',
str(citation_files['html'])
]
logger.info(
'Generating an HTML version of the citation boilerplate...')
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.html file:\n%s',
' '.join(cmd))
# Generate LaTex file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('fmriprep', 'data/boilerplate.bib'), '--natbib',
str(citation_files['md']), '-o',
str(citation_files['tex'])
]
logger.info(
'Generating a LaTeX version of the citation boilerplate...')
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.tex file:\n%s',
' '.join(cmd))
else:
copyfile(pkgrf('fmriprep', 'data/boilerplate.bib'),
citation_files['bib'])
else:
logger.warning(
'fMRIPrep could not find the markdown version of '
'the citation boilerplate (%s). HTML and LaTeX versions'
' of it will not be available', citation_files['md'])
# Generate reports phase
failed_reports = generate_reports(subject_list,
output_dir,
work_dir,
run_uuid,
packagename='fmriprep')
write_derivative_description(bids_dir, output_dir / 'fmriprep')
if failed_reports and not opts.notrack:
sentry_sdk.capture_message(
'Report generation failed for %d subjects' % failed_reports,
level='error')
sys.exit(int((errno + failed_reports) > 0))
def build_workflow(opts, retval):
"""
Create the Nipype Workflow that supports the whole execution
graph, given the inputs.
All the checks and the construction of the workflow are done
inside this function that has pickleable inputs and output
dictionary (``retval``) to allow isolation using a
``multiprocessing.Process`` that allows fmriprep to enforce
a hard-limited memory-scope.
"""
from bids import BIDSLayout
from nipype import logging as nlogging, config as ncfg
from niworkflows.utils.bids import collect_participants
from niworkflows.reports import generate_reports
from ..__about__ import __version__
from ..workflows.base import init_fmriprep_wf
build_log = nlogging.getLogger('nipype.workflow')
INIT_MSG = """
Running fMRIPREP version {version}:
* BIDS dataset path: {bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {uuid}.
""".format
bids_dir = opts.bids_dir.resolve()
output_dir = opts.output_dir.resolve()
work_dir = opts.work_dir.resolve()
retval['return_code'] = 1
retval['workflow'] = None
retval['bids_dir'] = str(bids_dir)
retval['output_dir'] = str(output_dir)
retval['work_dir'] = str(work_dir)
if output_dir == bids_dir:
build_log.error(
'The selected output folder is the same as the input BIDS folder. '
'Please modify the output path (suggestion: %s).', bids_dir /
'derivatives' / ('fmriprep-%s' % __version__.split('+')[0]))
retval['return_code'] = 1
return retval
output_spaces = parse_spaces(opts)
# Set up some instrumental utilities
run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4())
retval['run_uuid'] = run_uuid
# First check that bids_dir looks like a BIDS folder
layout = BIDSLayout(str(bids_dir),
validate=False,
ignore=("code", "stimuli", "sourcedata", "models",
"derivatives", re.compile(r'^\.')))
subject_list = collect_participants(
layout, participant_label=opts.participant_label)
retval['subject_list'] = subject_list
# Load base plugin_settings from file if --use-plugin
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
plugin_settings.setdefault('plugin_args', {})
else:
# Defaults
plugin_settings = {
'plugin': 'MultiProc',
'plugin_args': {
'raise_insufficient': False,
'maxtasksperchild': 1,
}
}
# Resource management options
# Note that we're making strong assumptions about valid plugin args
# This may need to be revisited if people try to use batch plugins
nthreads = plugin_settings['plugin_args'].get('n_procs')
# Permit overriding plugin config with specific CLI options
if nthreads is None or opts.nthreads is not None:
nthreads = opts.nthreads
if nthreads is None or nthreads < 1:
nthreads = cpu_count()
plugin_settings['plugin_args']['n_procs'] = nthreads
if opts.mem_mb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_mb / 1024
omp_nthreads = opts.omp_nthreads
if omp_nthreads == 0:
omp_nthreads = min(nthreads - 1 if nthreads > 1 else cpu_count(), 8)
if 1 < nthreads < omp_nthreads:
build_log.warning(
'Per-process threads (--omp-nthreads=%d) exceed total '
'threads (--nthreads/--n_cpus=%d)', omp_nthreads, nthreads)
retval['plugin_settings'] = plugin_settings
# Set up directories
log_dir = output_dir / 'fmriprep' / 'logs'
# Check and create output and working directories
output_dir.mkdir(exist_ok=True, parents=True)
log_dir.mkdir(exist_ok=True, parents=True)
work_dir.mkdir(exist_ok=True, parents=True)
# Nipype config (logs and execution)
ncfg.update_config({
'logging': {
'log_directory': str(log_dir),
'log_to_file': True
},
'execution': {
'crashdump_dir': str(log_dir),
'crashfile_format': 'txt',
'get_linked_libs': False,
'stop_on_first_crash': opts.stop_on_first_crash,
},
'monitoring': {
'enabled': opts.resource_monitor,
'sample_frequency': '0.5',
'summary_append': True,
}
})
if opts.resource_monitor:
ncfg.enable_resource_monitor()
# Called with reports only
if opts.reports_only:
build_log.log(25, 'Running --reports-only on participants %s',
', '.join(subject_list))
if opts.run_uuid is not None:
run_uuid = opts.run_uuid
retval['run_uuid'] = run_uuid
retval['return_code'] = generate_reports(subject_list,
output_dir,
work_dir,
run_uuid,
packagename='fmriprep')
return retval
# Build main workflow
build_log.log(
25,
INIT_MSG(version=__version__,
bids_dir=bids_dir,
subject_list=subject_list,
uuid=run_uuid))
retval['workflow'] = init_fmriprep_wf(
anat_only=opts.anat_only,
aroma_melodic_dim=opts.aroma_melodic_dimensionality,
bold2t1w_dof=opts.bold2t1w_dof,
cifti_output=opts.cifti_output,
debug=opts.sloppy,
dummy_scans=opts.dummy_scans,
echo_idx=opts.echo_idx,
err_on_aroma_warn=opts.error_on_aroma_warnings,
fmap_bspline=opts.fmap_bspline,
fmap_demean=opts.fmap_no_demean,
force_syn=opts.force_syn,
freesurfer=opts.run_reconall,
hires=opts.hires,
ignore=opts.ignore,
layout=layout,
longitudinal=opts.longitudinal,
low_mem=opts.low_mem,
medial_surface_nan=opts.medial_surface_nan,
omp_nthreads=omp_nthreads,
output_dir=str(output_dir),
output_spaces=output_spaces,
run_uuid=run_uuid,
regressors_all_comps=opts.return_all_components,
regressors_fd_th=opts.fd_spike_threshold,
regressors_dvars_th=opts.dvars_spike_threshold,
skull_strip_fixed_seed=opts.skull_strip_fixed_seed,
skull_strip_template=opts.skull_strip_template,
subject_list=subject_list,
t2s_coreg=opts.t2s_coreg,
task_id=opts.task_id,
use_aroma=opts.use_aroma,
use_bbr=opts.use_bbr,
use_syn=opts.use_syn_sdc,
work_dir=str(work_dir),
)
retval['return_code'] = 0
logs_path = Path(output_dir) / 'fmriprep' / 'logs'
boilerplate = retval['workflow'].visit_desc()
if boilerplate:
citation_files = {
ext: logs_path / ('CITATION.%s' % ext)
for ext in ('bib', 'tex', 'md', 'html')
}
# To please git-annex users and also to guarantee consistency
# among different renderings of the same file, first remove any
# existing one
for citation_file in citation_files.values():
try:
citation_file.unlink()
except FileNotFoundError:
pass
citation_files['md'].write_text(boilerplate)
build_log.log(
25, 'Works derived from this fMRIPrep execution should '
'include the following boilerplate:\n\n%s', boilerplate)
return retval
def build_workflow(opts, retval):
"""
Create the Nipype Workflow that supports the whole execution graph, given the inputs.
All the checks and the construction of the workflow are done
inside this function that has pickleable inputs and output
dictionary (``retval``) to allow isolation using a
``multiprocessing.Process`` that allows smriprep to enforce
a hard-limited memory-scope.
"""
from shutil import copyfile
from os import cpu_count
import uuid
from time import strftime
from subprocess import check_call, CalledProcessError, TimeoutExpired
from pkg_resources import resource_filename as pkgrf
import json
from bids import BIDSLayout
from nipype import logging, config as ncfg
from niworkflows.utils.bids import collect_participants
from ..__about__ import __version__
from ..workflows.base import init_smriprep_wf
logger = logging.getLogger('nipype.workflow')
INIT_MSG = """
Running sMRIPrep version {version}:
* BIDS dataset path: {bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {uuid}.
{spaces}
""".format
# Set up some instrumental utilities
run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4())
# First check that bids_dir looks like a BIDS folder
bids_dir = opts.bids_dir.resolve()
layout = BIDSLayout(str(bids_dir), validate=False)
subject_list = collect_participants(
layout, participant_label=opts.participant_label)
bids_filters = json.loads(
opts.bids_filter_file.read_text()) if opts.bids_filter_file else None
# Load base plugin_settings from file if --use-plugin
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
plugin_settings.setdefault('plugin_args', {})
else:
# Defaults
plugin_settings = {
'plugin': 'MultiProc',
'plugin_args': {
'raise_insufficient': False,
'maxtasksperchild': 1,
}
}
# Resource management options
# Note that we're making strong assumptions about valid plugin args
# This may need to be revisited if people try to use batch plugins
nprocs = plugin_settings['plugin_args'].get('n_procs')
# Permit overriding plugin config with specific CLI options
if nprocs is None or opts.nprocs is not None:
nprocs = opts.nprocs
if nprocs is None or nprocs < 1:
nprocs = cpu_count()
plugin_settings['plugin_args']['n_procs'] = nprocs
if opts.mem_gb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_gb
omp_nthreads = opts.omp_nthreads
if omp_nthreads == 0:
omp_nthreads = min(nprocs - 1 if nprocs > 1 else cpu_count(), 8)
if 1 < nprocs < omp_nthreads:
logger.warning(
'Per-process threads (--omp-nthreads=%d) exceed total '
'available CPUs (--nprocs/--ncpus=%d)', omp_nthreads, nprocs)
# Set up directories
output_dir = opts.output_dir.resolve()
log_dir = output_dir / 'smriprep' / 'logs'
work_dir = opts.work_dir.resolve()
# Check and create output and working directories
log_dir.mkdir(parents=True, exist_ok=True)
work_dir.mkdir(parents=True, exist_ok=True)
# Nipype config (logs and execution)
ncfg.update_config({
'logging': {
'log_directory': str(log_dir),
'log_to_file': True
},
'execution': {
'crashdump_dir': str(log_dir),
'crashfile_format': 'txt',
'get_linked_libs': False,
'stop_on_first_crash': opts.stop_on_first_crash,
},
'monitoring': {
'enabled': opts.resource_monitor,
'sample_frequency': '0.5',
'summary_append': True,
}
})
if opts.resource_monitor:
ncfg.enable_resource_monitor()
retval['return_code'] = 0
retval['plugin_settings'] = plugin_settings
retval['bids_dir'] = str(bids_dir)
retval['output_dir'] = str(output_dir)
retval['work_dir'] = str(work_dir)
retval['subject_list'] = subject_list
retval['run_uuid'] = run_uuid
retval['workflow'] = None
# Called with reports only
if opts.reports_only:
from niworkflows.reports import generate_reports
logger.log(25, 'Running --reports-only on participants %s',
', '.join(subject_list))
if opts.run_uuid is not None:
run_uuid = opts.run_uuid
retval['return_code'] = generate_reports(subject_list,
str(output_dir),
run_uuid,
packagename="smriprep")
return retval
logger.log(
25,
INIT_MSG(version=__version__,
bids_dir=bids_dir,
subject_list=subject_list,
uuid=run_uuid,
spaces=opts.output_spaces))
# Build main workflow
retval['workflow'] = init_smriprep_wf(
debug=opts.sloppy,
fast_track=opts.fast_track,
freesurfer=opts.run_reconall,
fs_subjects_dir=opts.fs_subjects_dir,
hires=opts.hires,
layout=layout,
longitudinal=opts.longitudinal,
low_mem=opts.low_mem,
omp_nthreads=omp_nthreads,
output_dir=str(output_dir),
run_uuid=run_uuid,
skull_strip_fixed_seed=opts.skull_strip_fixed_seed,
skull_strip_mode=opts.skull_strip_mode,
skull_strip_template=opts.skull_strip_template[0],
spaces=opts.output_spaces,
subject_list=subject_list,
work_dir=str(work_dir),
bids_filters=bids_filters,
)
retval['return_code'] = 0
boilerplate = retval['workflow'].visit_desc()
(log_dir / 'CITATION.md').write_text(boilerplate)
logger.log(
25, 'Works derived from this sMRIPrep execution should '
'include the following boilerplate:\n\n%s', boilerplate)
# Generate HTML file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('smriprep',
'data/boilerplate.bib'), '--filter', 'pandoc-citeproc',
'--metadata', 'pagetitle="sMRIPrep citation boilerplate"',
str(log_dir / 'CITATION.md'), '-o',
str(log_dir / 'CITATION.html')
]
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.html file:\n%s',
' '.join(cmd))
# Generate LaTex file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('smriprep', 'data/boilerplate.bib'), '--natbib',
str(log_dir / 'CITATION.md'), '-o',
str(log_dir / 'CITATION.tex')
]
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.tex file:\n%s',
' '.join(cmd))
else:
copyfile(pkgrf('smriprep', 'data/boilerplate.bib'),
str(log_dir / 'CITATION.bib'))
return retval
def main():
"""Entry point."""
import os
import sys
import gc
from multiprocessing import Process, Manager
from .parser import parse_args
from ..utils.bids import write_derivative_description
parse_args()
popylar = None
if not config.execution.notrack:
import popylar
from ..__about__ import __ga_id__
config.loggers.cli.info(
"Your usage of dmriprep is being recorded using popylar (https://popylar.github.io/). ", # noqa
"For details, see https://nipreps.github.io/dmriprep/usage.html. ",
"To opt out, call dmriprep with a `--notrack` flag")
popylar.track_event(__ga_id__, 'run', 'cli_run')
# CRITICAL Save the config to a file. This is necessary because the execution graph
# is built as a separate process to keep the memory footprint low. The most
# straightforward way to communicate with the child process is via the filesystem.
config_file = config.execution.work_dir / '.dmriprep.toml'
config.to_filename(config_file)
# CRITICAL Call build_workflow(config_file, retval) in a subprocess.
# Because Python on Linux does not ever free virtual memory (VM), running the
# workflow construction jailed within a process preempts excessive VM buildup.
with Manager() as mgr:
from .workflow import build_workflow
retval = mgr.dict()
p = Process(target=build_workflow, args=(str(config_file), retval))
p.start()
p.join()
retcode = p.exitcode or retval.get('return_code', 0)
dmriprep_wf = retval.get('workflow', None)
# CRITICAL Load the config from the file. This is necessary because the ``build_workflow``
# function executed constrained in a process may change the config (and thus the global
# state of dMRIPrep).
config.load(config_file)
if config.execution.reports_only:
sys.exit(int(retcode > 0))
if dmriprep_wf and config.execution.write_graph:
dmriprep_wf.write_graph(graph2use="colored",
format='svg',
simple_form=True)
retcode = retcode or (dmriprep_wf is None) * os.EX_SOFTWARE
if retcode != 0:
sys.exit(retcode)
# Generate boilerplate
with Manager() as mgr:
from .workflow import build_boilerplate
p = Process(target=build_boilerplate,
args=(str(config_file), dmriprep_wf))
p.start()
p.join()
if config.execution.boilerplate_only:
sys.exit(int(retcode > 0))
# Clean up master process before running workflow, which may create forks
gc.collect()
if popylar is not None:
popylar.track_event(__ga_id__, 'run', 'started')
config.loggers.workflow.log(
15, '\n'.join(['dMRIPrep config:'] +
['\t\t%s' % s for s in config.dumps().splitlines()]))
config.loggers.workflow.log(25, 'dMRIPrep started!')
errno = 1 # Default is error exit unless otherwise set
try:
dmriprep_wf.run(**config.nipype.get_plugin())
except Exception as e:
if not config.execution.notrack:
popylar.track_event(__ga_id__, 'run', 'error')
config.loggers.workflow.critical('dMRIPrep failed: %s', e)
raise
else:
config.loggers.workflow.log(25, 'dMRIPrep finished successfully!')
# Bother users with the boilerplate only iff the workflow went okay.
if (config.execution.output_dir / 'dmriprep' / 'logs' /
'CITATION.md').exists():
config.loggers.workflow.log(
25, 'Works derived from this dMRIPrep execution should '
'include the following boilerplate:\n\n%s',
(config.execution.output_dir / 'dmriprep' / 'logs' /
'CITATION.md').read_text())
if config.workflow.run_reconall:
from templateflow import api
from niworkflows.utils.misc import _copy_any
dseg_tsv = str(
api.get('fsaverage', suffix='dseg', extension=['.tsv']))
_copy_any(
dseg_tsv,
str(config.execution.output_dir / 'dmriprep' /
'desc-aseg_dseg.tsv'))
_copy_any(
dseg_tsv,
str(config.execution.output_dir / 'dmriprep' /
'desc-aparcaseg_dseg.tsv'))
errno = 0
finally:
from niworkflows.reports import generate_reports
from pkg_resources import resource_filename as pkgrf
# Generate reports phase
failed_reports = generate_reports(config.execution.participant_label,
config.execution.output_dir,
config.execution.work_dir,
config.execution.run_uuid,
config=pkgrf(
'dmriprep',
'config/reports-spec.yml'),
packagename='dmriprep')
write_derivative_description(config.execution.bids_dir,
config.execution.output_dir / 'dmriprep')
if failed_reports and not config.execution.notrack:
popylar.track_event(__ga_id__, 'run', 'reporting_error')
sys.exit(int((errno + failed_reports) > 0))
def main():
"""Entry point."""
import os
import sys
import gc
from multiprocessing import Process, Manager
from .parser import parse_args
from ..utils.bids import write_derivative_description
parse_args()
sentry_sdk = None
if not config.execution.notrack:
import sentry_sdk
from ..utils.sentry import sentry_setup
sentry_setup()
# CRITICAL Save the config to a file. This is necessary because the execution graph
# is built as a separate process to keep the memory footprint low. The most
# straightforward way to communicate with the child process is via the filesystem.
config_file = config.execution.work_dir / '.fmriprep.toml'
config.to_filename(config_file)
# CRITICAL Call build_workflow(config_file, retval) in a subprocess.
# Because Python on Linux does not ever free virtual memory (VM), running the
# workflow construction jailed within a process preempts excessive VM buildup.
with Manager() as mgr:
from .workflow import build_workflow
retval = mgr.dict()
p = Process(target=build_workflow, args=(str(config_file), retval))
p.start()
p.join()
retcode = p.exitcode or retval.get('return_code', 0)
fmriprep_wf = retval.get('workflow', None)
# CRITICAL Load the config from the file. This is necessary because the ``build_workflow``
# function executed constrained in a process may change the config (and thus the global
# state of fMRIPrep).
config.load(config_file)
if config.execution.reports_only:
sys.exit(int(retcode > 0))
if fmriprep_wf and config.execution.write_graph:
fmriprep_wf.write_graph(graph2use="colored",
format='svg',
simple_form=True)
retcode = retcode or (fmriprep_wf is None) * os.EX_SOFTWARE
if retcode != 0:
sys.exit(retcode)
# Generate boilerplate
with Manager() as mgr:
from .workflow import build_boilerplate
p = Process(target=build_boilerplate,
args=(str(config_file), fmriprep_wf))
p.start()
p.join()
if config.execution.boilerplate_only:
sys.exit(int(retcode > 0))
# Clean up master process before running workflow, which may create forks
gc.collect()
# Sentry tracking
if sentry_sdk is not None:
with sentry_sdk.configure_scope() as scope:
scope.set_tag('run_uuid', config.execution.run_uuid)
scope.set_tag('npart', len(config.execution.participant_label))
sentry_sdk.add_breadcrumb(message='fMRIPrep started', level='info')
sentry_sdk.capture_message('fMRIPrep started', level='info')
config.loggers.workflow.log(
15, '\n'.join(['fMRIPrep config:'] +
['\t\t%s' % s for s in config.dumps().splitlines()]))
config.loggers.workflow.log(25, 'fMRIPrep started!')
errno = 1 # Default is error exit unless otherwise set
try:
fmriprep_wf.run(**config.nipype.get_plugin())
except Exception as e:
if not config.execution.notrack:
from ..utils.sentry import process_crashfile
crashfolders = [
config.execution.output_dir / 'fmriprep' / 'sub-{}'.format(s) /
'log' / config.execution.run_uuid
for s in config.execution.participant_label
]
for crashfolder in crashfolders:
for crashfile in crashfolder.glob('crash*.*'):
process_crashfile(crashfile)
if "Workflow did not execute cleanly" not in str(e):
sentry_sdk.capture_exception(e)
config.loggers.workflow.critical('fMRIPrep failed: %s', e)
raise
else:
config.loggers.workflow.log(25, 'fMRIPrep finished successfully!')
if not config.execution.notrack:
success_message = 'fMRIPrep finished without errors'
sentry_sdk.add_breadcrumb(message=success_message, level='info')
sentry_sdk.capture_message(success_message, level='info')
# Bother users with the boilerplate only iff the workflow went okay.
if (config.execution.output_dir / 'fmriprep' / 'logs' /
'CITATION.md').exists():
config.loggers.workflow.log(
25, 'Works derived from this fMRIPrep execution should '
'include the following boilerplate:\n\n%s',
(config.execution.output_dir / 'fmriprep' / 'logs' /
'CITATION.md').read_text())
if config.workflow.run_reconall:
from templateflow import api
from niworkflows.utils.misc import _copy_any
dseg_tsv = str(
api.get('fsaverage', suffix='dseg', extension=['.tsv']))
_copy_any(
dseg_tsv,
str(config.execution.output_dir / 'fmriprep' /
'desc-aseg_dseg.tsv'))
_copy_any(
dseg_tsv,
str(config.execution.output_dir / 'fmriprep' /
'desc-aparcaseg_dseg.tsv'))
errno = 0
finally:
from niworkflows.reports import generate_reports
from pkg_resources import resource_filename as pkgrf
# Generate reports phase
failed_reports = generate_reports(config.execution.participant_label,
config.execution.output_dir,
config.execution.work_dir,
config.execution.run_uuid,
config=pkgrf(
'fmriprep',
'data/reports-spec.yml'),
packagename='fmriprep')
write_derivative_description(config.execution.bids_dir,
config.execution.output_dir / 'fmriprep')
if failed_reports and not config.execution.notrack:
sentry_sdk.capture_message(
'Report generation failed for %d subjects' % failed_reports,
level='error')
sys.exit(int((errno + failed_reports) > 0))
def build_opts(opts):
"""Trigger a new process that builds the workflow graph, based on the input options."""
import os
from pathlib import Path
import logging
import sys
import gc
import warnings
from multiprocessing import set_start_method, Process, Manager
from nipype import logging as nlogging
from niworkflows.utils.misc import check_valid_fs_license
set_start_method('forkserver')
logging.addLevelName(
25, 'IMPORTANT') # Add a new level between INFO and WARNING
logging.addLevelName(15,
'VERBOSE') # Add a new level between INFO and DEBUG
logger = logging.getLogger('cli')
def _warn_redirect(message,
category,
filename,
lineno,
file=None,
line=None):
logger.warning('Captured warning (%s): %s', category, message)
warnings.showwarning = _warn_redirect
# Precedence: --fs-license-file, $FS_LICENSE, default_license
if opts.fs_license_file is not None:
os.environ["FS_LICENSE"] = os.path.abspath(opts.fs_license_file)
if not check_valid_fs_license():
raise RuntimeError(
'ERROR: a valid license file is required for FreeSurfer to run. '
'sMRIPrep looked for an existing license file at several paths, in this '
'order: 1) command line argument ``--fs-license-file``; 2) ``$FS_LICENSE`` '
'environment variable; and 3) the ``$FREESURFER_HOME/license.txt`` path. '
'Get it (for free) by registering at https://'
'surfer.nmr.mgh.harvard.edu/registration.html')
# Retrieve logging level
log_level = int(max(25 - 5 * opts.verbose_count, logging.DEBUG))
# Set logging
logger.setLevel(log_level)
nlogging.getLogger('nipype.workflow').setLevel(log_level)
nlogging.getLogger('nipype.interface').setLevel(log_level)
nlogging.getLogger('nipype.utils').setLevel(log_level)
errno = 0
# Call build_workflow(opts, retval)
with Manager() as mgr:
retval = mgr.dict()
p = Process(target=build_workflow, args=(opts, retval))
p.start()
p.join()
if p.exitcode != 0:
sys.exit(p.exitcode)
smriprep_wf = retval['workflow']
plugin_settings = retval['plugin_settings']
bids_dir = retval['bids_dir']
output_dir = retval['output_dir']
subject_list = retval['subject_list']
run_uuid = retval['run_uuid']
retcode = retval['return_code']
if smriprep_wf is None:
sys.exit(1)
if opts.write_graph:
smriprep_wf.write_graph(graph2use="colored",
format='svg',
simple_form=True)
if opts.reports_only:
sys.exit(int(retcode > 0))
if opts.boilerplate:
sys.exit(int(retcode > 0))
# Check workflow for missing commands
missing = check_deps(smriprep_wf)
if missing:
print("Cannot run sMRIPrep. Missing dependencies:")
for iface, cmd in missing:
print("\t{} (Interface: {})".format(cmd, iface))
sys.exit(2)
# Clean up master process before running workflow, which may create forks
gc.collect()
try:
smriprep_wf.run(**plugin_settings)
except RuntimeError:
errno = 1
else:
if opts.run_reconall:
from templateflow import api
from niworkflows.utils.misc import _copy_any
dseg_tsv = str(
api.get('fsaverage', suffix='dseg', extension=['.tsv']))
_copy_any(
dseg_tsv,
str(Path(output_dir) / 'smriprep' / 'desc-aseg_dseg.tsv'))
_copy_any(
dseg_tsv,
str(Path(output_dir) / 'smriprep' / 'desc-aparcaseg_dseg.tsv'))
logger.log(25, 'sMRIPrep finished without errors')
finally:
from niworkflows.reports import generate_reports
from ..utils.bids import write_derivative_description
logger.log(25, 'Writing reports for participants: %s',
', '.join(subject_list))
# Generate reports phase
errno += generate_reports(subject_list,
output_dir,
run_uuid,
packagename='smriprep')
write_derivative_description(bids_dir,
str(Path(output_dir) / 'smriprep'))
sys.exit(int(errno > 0))
def build_workflow(config_file, retval):
"""Create the Nipype Workflow that supports the whole execution graph."""
from niworkflows.utils.bids import collect_participants, check_pipeline_version
from niworkflows.reports import generate_reports
from .. import config
from ..utils.misc import check_deps
from ..workflows.base import init_fmriprep_wf
config.load(config_file)
build_log = config.loggers.workflow
output_dir = config.execution.output_dir
version = config.environment.version
retval["return_code"] = 1
retval["workflow"] = None
# warn if older results exist: check for dataset_description.json in output folder
msg = check_pipeline_version(
version, output_dir / "fmriprep" / "dataset_description.json")
if msg is not None:
build_log.warning(msg)
# Please note this is the input folder's dataset_description.json
dset_desc_path = config.execution.bids_dir / "dataset_description.json"
if dset_desc_path.exists():
from hashlib import sha256
desc_content = dset_desc_path.read_bytes()
config.execution.bids_description_hash = sha256(
desc_content).hexdigest()
# First check that bids_dir looks like a BIDS folder
subject_list = collect_participants(
config.execution.layout,
participant_label=config.execution.participant_label)
# Called with reports only
if config.execution.reports_only:
from pkg_resources import resource_filename as pkgrf
build_log.log(25, "Running --reports-only on participants %s",
", ".join(subject_list))
retval["return_code"] = generate_reports(
subject_list,
config.execution.output_dir,
config.execution.run_uuid,
config=pkgrf("fmriprep", "data/reports-spec.yml"),
packagename="fmriprep",
)
return retval
# Build main workflow
init_msg = f"""
Running fMRIPREP version {config.environment.version}:
* BIDS dataset path: {config.execution.bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {config.execution.run_uuid}.
* Output spaces: {config.execution.output_spaces}."""
if config.execution.anat_derivatives:
init_msg += f"""
* Anatomical derivatives: {config.execution.anat_derivatives}."""
if config.execution.fs_subjects_dir:
init_msg += f"""
* Pre-run FreeSurfer's SUBJECTS_DIR: {config.execution.fs_subjects_dir}."""
build_log.log(25, init_msg)
retval["workflow"] = init_fmriprep_wf()
# Check workflow for missing commands
missing = check_deps(retval["workflow"])
if missing:
build_log.critical(
"Cannot run fMRIPrep. Missing dependencies:%s",
"\n\t* ".join(
[""] +
[f"{cmd} (Interface: {iface})" for iface, cmd in missing]),
)
retval["return_code"] = 127 # 127 == command not found.
return retval
config.to_filename(config_file)
build_log.info(
"fMRIPrep workflow graph with %d nodes built successfully.",
len(retval["workflow"]._get_all_nodes()),
)
retval["return_code"] = 0
return retval
def build_workflow(config_file, retval):
"""Create the Nipype Workflow that supports the whole execution graph."""
from niworkflows.utils.bids import collect_participants, check_pipeline_version
from niworkflows.utils.misc import check_valid_fs_license
from niworkflows.reports import generate_reports
from .. import config
from ..utils.misc import check_deps
from ..workflows.base import init_fmriprep_wf
config.load(config_file)
build_log = config.loggers.workflow
output_dir = config.execution.output_dir
version = config.environment.version
retval["return_code"] = 1
retval["workflow"] = None
# warn if older results exist: check for dataset_description.json in output folder
msg = check_pipeline_version(
version, output_dir / "fmriprep-rodents" / "dataset_description.json")
if msg is not None:
build_log.warning(msg)
# Please note this is the input folder's dataset_description.json
dset_desc_path = config.execution.bids_dir / "dataset_description.json"
if dset_desc_path.exists():
from hashlib import sha256
desc_content = dset_desc_path.read_bytes()
config.execution.bids_description_hash = sha256(
desc_content).hexdigest()
# First check that bids_dir looks like a BIDS folder
subject_list = collect_participants(
config.execution.layout,
participant_label=config.execution.participant_label)
# Called with reports only
if config.execution.reports_only:
from pkg_resources import resource_filename as pkgrf
build_log.log(25, "Running --reports-only on participants %s",
", ".join(subject_list))
retval["return_code"] = generate_reports(
subject_list,
config.execution.output_dir,
config.execution.run_uuid,
config=pkgrf("fmriprep_rodents", "data/reports-spec.yml"),
packagename="fmriprep-rodents",
)
return retval
# Build main workflow
init_msg = f"""
Running fMRIPREP version {config.environment.version}:
* BIDS dataset path: {config.execution.bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {config.execution.run_uuid}.
* Output spaces: {config.execution.output_spaces}."""
if config.execution.anat_derivatives:
init_msg += f"""
* Anatomical derivatives: {config.execution.anat_derivatives}."""
if config.execution.fs_subjects_dir:
init_msg += f"""
* Pre-run FreeSurfer's SUBJECTS_DIR: {config.execution.fs_subjects_dir}."""
build_log.log(25, init_msg)
retval["workflow"] = init_fmriprep_wf()
# Check for FS license after building the workflow
if not check_valid_fs_license():
build_log.critical("""\
ERROR: a valid license file is required for FreeSurfer to run. fMRIPrep looked for an existing \
license file at several paths, in this order: 1) command line argument ``--fs-license-file``; \
2) ``$FS_LICENSE`` environment variable; and 3) the ``$FREESURFER_HOME/license.txt`` path. Get it \
(for free) by registering at https://surfer.nmr.mgh.harvard.edu/registration.html"""
)
retval["return_code"] = 126 # 126 == Command invoked cannot execute.
return retval
# Check workflow for missing commands
missing = check_deps(retval["workflow"])
if missing:
build_log.critical(
"Cannot run fMRIPrep. Missing dependencies:%s",
"\n\t* ".join(
[""] +
[f"{cmd} (Interface: {iface})" for iface, cmd in missing]),
)
retval["return_code"] = 127 # 127 == command not found.
return retval
config.to_filename(config_file)
build_log.info(
"fMRIPrep workflow graph with %d nodes built successfully.",
len(retval["workflow"]._get_all_nodes()),
)
retval["return_code"] = 0
return retval
def main():
"""Entry point."""
from os import EX_SOFTWARE
from pathlib import Path
import sys
import gc
from multiprocessing import Process, Manager
from .parser import parse_args
from ..utils.bids import write_derivative_description
parse_args()
sentry_sdk = None
if not config.execution.notrack:
import sentry_sdk
from ..utils.sentry import sentry_setup
sentry_setup()
# CRITICAL Save the config to a file. This is necessary because the execution graph
# is built as a separate process to keep the memory footprint low. The most
# straightforward way to communicate with the child process is via the filesystem.
config_file = config.execution.work_dir / f"config-{config.execution.run_uuid}.toml"
config.to_filename(config_file)
# CRITICAL Call build_workflow(config_file, retval) in a subprocess.
# Because Python on Linux does not ever free virtual memory (VM), running the
# workflow construction jailed within a process preempts excessive VM buildup.
with Manager() as mgr:
from .workflow import build_workflow
retval = mgr.dict()
p = Process(target=build_workflow, args=(str(config_file), retval))
p.start()
p.join()
retcode = p.exitcode or retval.get("return_code", 0)
fmriprep_wf = retval.get("workflow", None)
# CRITICAL Load the config from the file. This is necessary because the ``build_workflow``
# function executed constrained in a process may change the config (and thus the global
# state of fMRIPrep).
config.load(config_file)
if config.execution.reports_only:
sys.exit(int(retcode > 0))
if fmriprep_wf and config.execution.write_graph:
fmriprep_wf.write_graph(graph2use="colored",
format="svg",
simple_form=True)
retcode = retcode or (fmriprep_wf is None) * EX_SOFTWARE
if retcode != 0:
sys.exit(retcode)
# Generate boilerplate
with Manager() as mgr:
from .workflow import build_boilerplate
p = Process(target=build_boilerplate,
args=(str(config_file), fmriprep_wf))
p.start()
p.join()
if config.execution.boilerplate_only:
sys.exit(int(retcode > 0))
# Clean up master process before running workflow, which may create forks
gc.collect()
# Sentry tracking
if sentry_sdk is not None:
with sentry_sdk.configure_scope() as scope:
scope.set_tag("run_uuid", config.execution.run_uuid)
scope.set_tag("npart", len(config.execution.participant_label))
sentry_sdk.add_breadcrumb(message="fMRIPrep started", level="info")
sentry_sdk.capture_message("fMRIPrep started", level="info")
config.loggers.workflow.log(
15,
"\n".join(["fMRIPrep config:"] +
["\t\t%s" % s for s in config.dumps().splitlines()]),
)
config.loggers.workflow.log(25, "fMRIPrep started!")
errno = 1 # Default is error exit unless otherwise set
try:
fmriprep_wf.run(**config.nipype.get_plugin())
except Exception as e:
if not config.execution.notrack:
from ..utils.sentry import process_crashfile
crashfolders = [
config.execution.output_dir / "fmriprep" / "sub-{}".format(s) /
"log" / config.execution.run_uuid
for s in config.execution.participant_label
]
for crashfolder in crashfolders:
for crashfile in crashfolder.glob("crash*.*"):
process_crashfile(crashfile)
if "Workflow did not execute cleanly" not in str(e):
sentry_sdk.capture_exception(e)
config.loggers.workflow.critical("fMRIPrep failed: %s", e)
raise
else:
config.loggers.workflow.log(25, "fMRIPrep finished successfully!")
if not config.execution.notrack:
success_message = "fMRIPrep finished without errors"
sentry_sdk.add_breadcrumb(message=success_message, level="info")
sentry_sdk.capture_message(success_message, level="info")
# Bother users with the boilerplate only iff the workflow went okay.
boiler_file = config.execution.output_dir / "fmriprep" / "logs" / "CITATION.md"
if boiler_file.exists():
if config.environment.exec_env in (
"singularity",
"docker",
"fmriprep-docker",
):
boiler_file = Path("<OUTPUT_PATH>") / boiler_file.relative_to(
config.execution.output_dir)
config.loggers.workflow.log(
25,
"Works derived from this fMRIPrep execution should include the "
f"boilerplate text found in {boiler_file}.",
)
if config.workflow.run_reconall:
from templateflow import api
from niworkflows.utils.misc import _copy_any
dseg_tsv = str(
api.get("fsaverage", suffix="dseg", extension=[".tsv"]))
_copy_any(
dseg_tsv,
str(config.execution.output_dir / "fmriprep" /
"desc-aseg_dseg.tsv"),
)
_copy_any(
dseg_tsv,
str(config.execution.output_dir / "fmriprep" /
"desc-aparcaseg_dseg.tsv"),
)
errno = 0
finally:
from niworkflows.reports import generate_reports
from pkg_resources import resource_filename as pkgrf
# Generate reports phase
failed_reports = generate_reports(
config.execution.participant_label,
config.execution.output_dir,
config.execution.run_uuid,
config=pkgrf("fmriprep", "data/reports-spec.yml"),
packagename="fmriprep",
)
write_derivative_description(config.execution.bids_dir,
config.execution.output_dir / "fmriprep")
if failed_reports and not config.execution.notrack:
sentry_sdk.capture_message(
"Report generation failed for %d subjects" % failed_reports,
level="error",
)
sys.exit(int((errno + failed_reports) > 0))
def build_workflow(config_file, retval):
"""Create the Nipype Workflow that supports the whole execution graph."""
from niworkflows.utils.bids import collect_participants, check_pipeline_version
from niworkflows.reports import generate_reports
from .. import config
from ..utils.misc import check_deps
from ..workflows.base import init_fmriprep_wf
config.load(config_file)
build_log = config.loggers.workflow
output_dir = config.execution.output_dir
version = config.environment.version
retval['return_code'] = 1
retval['workflow'] = None
# warn if older results exist: check for dataset_description.json in output folder
msg = check_pipeline_version(
version, output_dir / 'fmriprep' / 'dataset_description.json')
if msg is not None:
build_log.warning(msg)
# Please note this is the input folder's dataset_description.json
dset_desc_path = config.execution.bids_dir / 'dataset_description.json'
if dset_desc_path.exists():
from hashlib import sha256
desc_content = dset_desc_path.read_bytes()
config.execution.bids_description_hash = sha256(
desc_content).hexdigest()
# First check that bids_dir looks like a BIDS folder
subject_list = collect_participants(
config.execution.layout,
participant_label=config.execution.participant_label)
# Called with reports only
if config.execution.reports_only:
from pkg_resources import resource_filename as pkgrf
build_log.log(25, 'Running --reports-only on participants %s',
', '.join(subject_list))
retval['return_code'] = generate_reports(subject_list,
config.execution.output_dir,
config.execution.work_dir,
config.execution.run_uuid,
config=pkgrf(
'fmriprep',
'data/reports-spec.yml'),
packagename='fmriprep')
return retval
# Build main workflow
INIT_MSG = """
Running fMRIPREP version {version}:
* BIDS dataset path: {bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {uuid}.
* Output spaces: {spaces}.
""".format
build_log.log(
25,
INIT_MSG(version=config.environment.version,
bids_dir=config.execution.bids_dir,
subject_list=subject_list,
uuid=config.execution.run_uuid,
spaces=config.execution.output_spaces))
retval['workflow'] = init_fmriprep_wf()
# Check workflow for missing commands
missing = check_deps(retval['workflow'])
if missing:
build_log.critical(
"Cannot run fMRIPrep. Missing dependencies:%s", '\n\t* %s'.join([
"{} (Interface: {})".format(cmd, iface)
for iface, cmd in missing
]))
retval['return_code'] = 127 # 127 == command not found.
return retval
config.to_filename(config_file)
build_log.info("fMRIPrep workflow graph with %d nodes built successfully.",
len(retval['workflow']._get_all_nodes()))
retval['return_code'] = 0
return retval
def main():
"""Entry point"""
from nipype import logging as nlogging
from multiprocessing import set_start_method, Process, Manager
from ..utils.bids import write_derivative_description, validate_input_dir
from ..__about__ import __ga_id__
set_start_method('forkserver')
warnings.showwarning = _warn_redirect
opts = get_parser().parse_args()
exec_env = os.name
if not opts.notrack:
logger.info(
"Your usage of dmriprep is being recorded using popylar (https://popylar.github.io/). ", # noqa
"For details, see https://nipreps.github.io/dmriprep/usage.html. ",
"To opt out, call dmriprep with a `--notrack` flag")
import popylar
popylar.track_event(__ga_id__, 'run', 'cli_run')
# Validate inputs
if not opts.skip_bids_validation:
print(
"Making sure the input data is BIDS compliant (warnings can be ignored in most "
"cases).")
validate_input_dir(exec_env, opts.bids_dir, opts.participant_label)
# FreeSurfer license
default_license = str(Path(os.getenv('FREESURFER_HOME')) / 'license.txt')
# Precedence: --fs-license-file, $FS_LICENSE, default_license
license_file = opts.fs_license_file or Path(
os.getenv('FS_LICENSE', default_license))
if not license_file.exists():
raise RuntimeError("""\
ERROR: a valid license file is required for FreeSurfer to run. dMRIPrep looked for an existing \
license file at several paths, in this order: 1) command line argument ``--fs-license-file``; \
2) ``$FS_LICENSE`` environment variable; and 3) the ``$FREESURFER_HOME/license.txt`` path. Get it \
(for free) by registering at https://surfer.nmr.mgh.harvard.edu/registration.html"""
)
os.environ['FS_LICENSE'] = str(license_file.resolve())
# Retrieve logging level
log_level = int(max(25 - 5 * opts.verbose_count, logging.DEBUG))
# Set logging
logger.setLevel(log_level)
nlogging.getLogger('nipype.workflow').setLevel(log_level)
nlogging.getLogger('nipype.interface').setLevel(log_level)
nlogging.getLogger('nipype.utils').setLevel(log_level)
# Call build_workflow(opts, retval)
with Manager() as mgr:
retval = mgr.dict()
p = Process(target=build_workflow, args=(opts, retval))
p.start()
p.join()
retcode = p.exitcode or retval.get('return_code', 0)
bids_dir = Path(retval.get('bids_dir'))
output_dir = Path(retval.get('output_dir'))
work_dir = Path(retval.get('work_dir'))
plugin_settings = retval.get('plugin_settings', None)
subject_list = retval.get('subject_list', None)
dmriprep_wf = retval.get('workflow', None)
run_uuid = retval.get('run_uuid', None)
if opts.reports_only:
sys.exit(int(retcode > 0))
if opts.boilerplate:
sys.exit(int(retcode > 0))
if dmriprep_wf and opts.write_graph:
dmriprep_wf.write_graph(graph2use="colored",
format='svg',
simple_form=True)
retcode = retcode or int(dmriprep_wf is None)
if retcode != 0:
sys.exit(retcode)
# Check workflow for missing commands
missing = check_deps(dmriprep_wf)
if missing:
print("Cannot run dMRIPrep. Missing dependencies:", file=sys.stderr)
for iface, cmd in missing:
print("\t{} (Interface: {})".format(cmd, iface))
sys.exit(2)
# Clean up master process before running workflow, which may create forks
gc.collect()
errno = 1 # Default is error exit unless otherwise set
try:
dmriprep_wf.run(**plugin_settings)
except Exception as e:
if not opts.notrack:
popylar.track_event(__ga_id__, 'run', 'cli_error')
logger.critical('dMRIPrep failed: %s', e)
raise e
else:
if opts.run_reconall:
from templateflow import api
from niworkflows.utils.misc import _copy_any
dseg_tsv = str(
api.get('fsaverage', suffix='dseg', extension=['.tsv']))
_copy_any(dseg_tsv,
str(output_dir / 'dmriprep' / 'desc-aseg_dseg.tsv'))
_copy_any(dseg_tsv,
str(output_dir / 'dmriprep' / 'desc-aparcaseg_dseg.tsv'))
errno = 0
logger.log(25, 'dMRIPrep finished without errors')
if not opts.notrack:
popylar.track_event(__ga_id__, 'run', 'cli_finished')
finally:
from niworkflows.reports import generate_reports
from subprocess import check_call, CalledProcessError, TimeoutExpired
from pkg_resources import resource_filename as pkgrf
from shutil import copyfile
citation_files = {
ext: output_dir / 'dmriprep' / 'logs' / ('CITATION.%s' % ext)
for ext in ('bib', 'tex', 'md', 'html')
}
if citation_files['md'].exists():
# Generate HTML file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('dmriprep',
'data/boilerplate.bib'), '--filter', 'pandoc-citeproc',
'--metadata', 'pagetitle="dMRIPrep citation boilerplate"',
str(citation_files['md']), '-o',
str(citation_files['html'])
]
logger.info(
'Generating an HTML version of the citation boilerplate...')
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.html file:\n%s',
' '.join(cmd))
# Generate LaTex file resolving citations
cmd = [
'pandoc', '-s', '--bibliography',
pkgrf('dmriprep', 'data/boilerplate.bib'), '--natbib',
str(citation_files['md']), '-o',
str(citation_files['tex'])
]
logger.info(
'Generating a LaTeX version of the citation boilerplate...')
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning('Could not generate CITATION.tex file:\n%s',
' '.join(cmd))
else:
copyfile(pkgrf('dmriprep', 'data/boilerplate.bib'),
citation_files['bib'])
else:
logger.warning(
'dMRIPrep could not find the markdown version of '
'the citation boilerplate (%s). HTML and LaTeX versions'
' of it will not be available', citation_files['md'])
# Generate reports phase
failed_reports = generate_reports(subject_list,
output_dir,
work_dir,
run_uuid,
config=pkgrf(
'dmriprep',
'config/reports-spec.yml'),
packagename='dmriprep')
write_derivative_description(bids_dir, output_dir / 'dmriprep')
sys.exit(int((errno + failed_reports) > 0))
def build_workflow(opts, retval):
"""
Create the Nipype Workflow that supports the whole execution graph, given the inputs.
All the checks and the construction of the workflow are done
inside this function that has pickleable inputs and output
dictionary (``retval``) to allow isolation using a
``multiprocessing.Process`` that allows smriprep to enforce
a hard-limited memory-scope.
"""
from shutil import copyfile
from os import cpu_count
import uuid
from time import strftime
from subprocess import check_call, CalledProcessError, TimeoutExpired
from pkg_resources import resource_filename as pkgrf
import json
from bids import BIDSLayout
from nipype import logging, config as ncfg
from niworkflows.utils.bids import collect_participants
from ..__about__ import __version__
from ..workflows.base import init_smriprep_wf
logger = logging.getLogger("nipype.workflow")
INIT_MSG = """
Running sMRIPrep version {version}:
* BIDS dataset path: {bids_dir}.
* Participant list: {subject_list}.
* Run identifier: {uuid}.
{spaces}
""".format
# Set up some instrumental utilities
run_uuid = "%s_%s" % (strftime("%Y%m%d-%H%M%S"), uuid.uuid4())
# First check that bids_dir looks like a BIDS folder
bids_dir = opts.bids_dir.resolve()
layout = BIDSLayout(str(bids_dir), validate=False)
subject_list = collect_participants(
layout, participant_label=opts.participant_label)
bids_filters = (json.loads(opts.bids_filter_file.read_text())
if opts.bids_filter_file else None)
# Load base plugin_settings from file if --use-plugin
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
plugin_settings.setdefault("plugin_args", {})
else:
# Defaults
plugin_settings = {
"plugin": "MultiProc",
"plugin_args": {
"raise_insufficient": False,
"maxtasksperchild": 1,
},
}
# Resource management options
# Note that we're making strong assumptions about valid plugin args
# This may need to be revisited if people try to use batch plugins
nprocs = plugin_settings["plugin_args"].get("n_procs")
# Permit overriding plugin config with specific CLI options
if nprocs is None or opts.nprocs is not None:
nprocs = opts.nprocs
if nprocs is None or nprocs < 1:
nprocs = cpu_count()
plugin_settings["plugin_args"]["n_procs"] = nprocs
if opts.mem_gb:
plugin_settings["plugin_args"]["memory_gb"] = opts.mem_gb
omp_nthreads = opts.omp_nthreads
if omp_nthreads == 0:
omp_nthreads = min(nprocs - 1 if nprocs > 1 else cpu_count(), 8)
if 1 < nprocs < omp_nthreads:
logger.warning(
"Per-process threads (--omp-nthreads=%d) exceed total "
"available CPUs (--nprocs/--ncpus=%d)",
omp_nthreads,
nprocs,
)
# Set up directories
output_dir = opts.output_dir.resolve()
log_dir = output_dir / "smriprep" / "logs"
work_dir = opts.work_dir.resolve()
# Check and create output and working directories
log_dir.mkdir(parents=True, exist_ok=True)
work_dir.mkdir(parents=True, exist_ok=True)
# Nipype config (logs and execution)
ncfg.update_config({
"logging": {
"log_directory": str(log_dir),
"log_to_file": True
},
"execution": {
"crashdump_dir": str(log_dir),
"crashfile_format": "txt",
"get_linked_libs": False,
"stop_on_first_crash": opts.stop_on_first_crash,
},
"monitoring": {
"enabled": opts.resource_monitor,
"sample_frequency": "0.5",
"summary_append": True,
},
})
if opts.resource_monitor:
ncfg.enable_resource_monitor()
retval["return_code"] = 0
retval["plugin_settings"] = plugin_settings
retval["bids_dir"] = str(bids_dir)
retval["output_dir"] = str(output_dir)
retval["work_dir"] = str(work_dir)
retval["subject_list"] = subject_list
retval["run_uuid"] = run_uuid
retval["workflow"] = None
# Called with reports only
if opts.reports_only:
from niworkflows.reports import generate_reports
logger.log(25, "Running --reports-only on participants %s",
", ".join(subject_list))
if opts.run_uuid is not None:
run_uuid = opts.run_uuid
retval["return_code"] = generate_reports(subject_list,
str(output_dir),
run_uuid,
packagename="smriprep")
return retval
output_spaces = opts.output_spaces
if not output_spaces.is_cached():
output_spaces.checkpoint()
logger.log(
25,
INIT_MSG(
version=__version__,
bids_dir=bids_dir,
subject_list=subject_list,
uuid=run_uuid,
spaces=output_spaces,
),
)
# Build main workflow
retval["workflow"] = init_smriprep_wf(
debug=opts.sloppy,
fast_track=opts.fast_track,
freesurfer=opts.run_reconall,
fs_subjects_dir=opts.fs_subjects_dir,
hires=opts.hires,
layout=layout,
longitudinal=opts.longitudinal,
low_mem=opts.low_mem,
omp_nthreads=omp_nthreads,
output_dir=str(output_dir),
run_uuid=run_uuid,
skull_strip_fixed_seed=opts.skull_strip_fixed_seed,
skull_strip_mode=opts.skull_strip_mode,
skull_strip_template=opts.skull_strip_template[0],
spaces=output_spaces,
subject_list=subject_list,
work_dir=str(work_dir),
bids_filters=bids_filters,
)
retval["return_code"] = 0
boilerplate = retval["workflow"].visit_desc()
(log_dir / "CITATION.md").write_text(boilerplate)
logger.log(
25,
"Works derived from this sMRIPrep execution should "
"include the following boilerplate:\n\n%s",
boilerplate,
)
# Generate HTML file resolving citations
cmd = [
"pandoc",
"-s",
"--bibliography",
pkgrf("smriprep", "data/boilerplate.bib"),
"--citeproc",
"--metadata",
'pagetitle="sMRIPrep citation boilerplate"',
str(log_dir / "CITATION.md"),
"-o",
str(log_dir / "CITATION.html"),
]
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning("Could not generate CITATION.html file:\n%s",
" ".join(cmd))
# Generate LaTex file resolving citations
cmd = [
"pandoc",
"-s",
"--bibliography",
pkgrf("smriprep", "data/boilerplate.bib"),
"--natbib",
str(log_dir / "CITATION.md"),
"-o",
str(log_dir / "CITATION.tex"),
]
try:
check_call(cmd, timeout=10)
except (FileNotFoundError, CalledProcessError, TimeoutExpired):
logger.warning("Could not generate CITATION.tex file:\n%s",
" ".join(cmd))
else:
copyfile(pkgrf("smriprep", "data/boilerplate.bib"),
str(log_dir / "CITATION.bib"))
return retval
def build_workflow(opts, retval):
from nipype import logging as nlogging, config as ncfg
from niworkflows.utils.bids import collect_participants
from niworkflows.reports import generate_reports
from ..__about__ import __version__
from time import strftime
import uuid
from ..workflows.base import init_base_wf
build_log = nlogging.getLogger('nipype.workflow')
INIT_MSG = """
# Running atlasTransform version {version}:
# * BIDS dataset path: {bids_dir}.
# * Participant list: {subject_list}.
# * Run identifier: {uuid}.
# """.format
pass
bids_dir = opts.bids_dir.resolve()
output_dir = opts.output_dir.resolve()
work_dir = opts.work_dir.resolve()
retval['return_code'] = 1
retval['workflow'] = None
retval['bids_dir'] = str(bids_dir)
retval['output_dir'] = str(output_dir)
retval['work_dir'] = str(work_dir)
if output_dir == bids_dir:
build_log.error(
'The selected output folder is the same as the input BIDS folder. '
'Please modify the output path (suggestion: %s).', bids_dir /
'derivatives' / ('atlasTransform-%s' % __version__.split('+')[0]))
retval['return_code'] = 1
return retval
# Set up some instrumental utilities
run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid.uuid4())
retval['run_uuid'] = run_uuid
# First check that bids_dir looks like a BIDS folder
layout = BIDSLayout(str(bids_dir),
validate=False,
ignore=("code", "stimuli", "sourcedata", "models",
"derivatives", re.compile(r'^\.')))
subject_list = collect_participants(
layout, participant_label=opts.participant_label)
retval['subject_list'] = subject_list
# Load base plugin_settings from file if --use-plugin
if opts.use_plugin is not None:
from yaml import load as loadyml
with open(opts.use_plugin) as f:
plugin_settings = loadyml(f)
plugin_settings.setdefault('plugin_args', {})
else:
# Defaults
plugin_settings = {
'plugin': 'MultiProc',
'plugin_args': {
'raise_insufficient': False,
'maxtasksperchild': 1,
}
}
# Resource management options
# Note that we're making strong assumptions about valid plugin args
# This may need to be revisited if people try to use batch plugins
nthreads = plugin_settings['plugin_args'].get('n_procs')
# Permit overriding plugin config with specific CLI options
if nthreads is None or opts.nthreads is not None:
nthreads = opts.nthreads
if nthreads is None or nthreads < 1:
nthreads = cpu_count()
plugin_settings['plugin_args']['n_procs'] = nthreads
if opts.mem_mb:
plugin_settings['plugin_args']['memory_gb'] = opts.mem_mb / 1024
omp_nthreads = opts.omp_nthreads
if omp_nthreads == 0:
omp_nthreads = min(nthreads - 1 if nthreads > 1 else cpu_count(), 8)
if 1 < nthreads < omp_nthreads:
build_log.warning(
'Per-process threads (--omp-nthreads=%d) exceed total '
'threads (--nthreads/--n_cpus=%d)', omp_nthreads, nthreads)
retval['plugin_settings'] = plugin_settings
# Set up directories
log_dir = output_dir / 'atlasTransform' / 'logs'
# Check and create output and working directories
output_dir.mkdir(exist_ok=True, parents=True)
log_dir.mkdir(exist_ok=True, parents=True)
work_dir.mkdir(exist_ok=True, parents=True)
# Nipype config (logs and execution)
ncfg.update_config({
'logging': {
'log_directory': str(log_dir),
'log_to_file': True
},
'execution': {
'crashdump_dir': str(log_dir),
'crashfile_format': 'txt',
'get_linked_libs': False,
'stop_on_first_crash': opts.stop_on_first_crash,
},
'monitoring': {
'enabled': opts.resource_monitor,
'sample_frequency': '0.5',
'summary_append': True,
}
})
if opts.resource_monitor:
ncfg.enable_resource_monitor()
# Called with reports only
if opts.reports_only:
build_log.log(25, 'Running --reports-only on participants %s',
', '.join(subject_list))
if opts.run_uuid is not None:
run_uuid = opts.run_uuid
retval['run_uuid'] = run_uuid
retval['return_code'] = generate_reports(subject_list,
output_dir,
work_dir,
run_uuid,
packagename='atlasTransform')
return retval
# Build main workflow
build_log.log(
25,
INIT_MSG(version=__version__,
bids_dir=bids_dir,
subject_list=subject_list,
uuid=run_uuid))
retval['workflow'] = init_base_wf(
opts=opts,
layout=layout,
run_uuid=run_uuid,
subject_list=subject_list,
work_dir=str(work_dir),
output_dir=str(output_dir),
)
retval['return_code'] = 0
logs_path = Path(output_dir) / 'atlasTransform' / 'logs'
boilerplate = retval['workflow'].visit_desc()
if boilerplate:
citation_files = {
ext: logs_path / ('CITATION.%s' % ext)
for ext in ('bib', 'tex', 'md', 'html')
}
# To please git-annex users and also to guarantee consistency
# among different renderings of the same file, first remove any
# existing one
for citation_file in citation_files.values():
try:
citation_file.unlink()
except FileNotFoundError:
pass
citation_files['md'].write_text(boilerplate)
build_log.log(
25, 'Works derived from this atlasTransform execution should '
'include the following boilerplate:\n\n%s', boilerplate)
return retval